WebAssembly http(s) transport layer

.github/workflows/test-wasm.yml

@@ -48,5 +48,7 @@ jobs:
       - name: Run WebAssembly tests
         shell: bash -l {0}
         working-directory: wasm
+        env:
+          GIT2CPP_TEST_PRIVATE_TOKEN: ${{ secrets.GIT2CPP_TEST_PRIVATE_TOKEN }}
         run: |
           make test

CMakeLists.txt

@@ -98,6 +98,18 @@ set(GIT2CPP_SRC
     ${GIT2CPP_SOURCE_DIR}/utils/progress.hpp
     ${GIT2CPP_SOURCE_DIR}/utils/terminal_pager.cpp
     ${GIT2CPP_SOURCE_DIR}/utils/terminal_pager.hpp
+    ${GIT2CPP_SOURCE_DIR}/wasm/libgit2_internals.cpp
+    ${GIT2CPP_SOURCE_DIR}/wasm/libgit2_internals.hpp
+    ${GIT2CPP_SOURCE_DIR}/wasm/response.cpp
+    ${GIT2CPP_SOURCE_DIR}/wasm/response.hpp
+    ${GIT2CPP_SOURCE_DIR}/wasm/scope.cpp
+    ${GIT2CPP_SOURCE_DIR}/wasm/scope.hpp
+    ${GIT2CPP_SOURCE_DIR}/wasm/stream.cpp
+    ${GIT2CPP_SOURCE_DIR}/wasm/stream.hpp
+    ${GIT2CPP_SOURCE_DIR}/wasm/subtransport.cpp
+    ${GIT2CPP_SOURCE_DIR}/wasm/subtransport.hpp
+    ${GIT2CPP_SOURCE_DIR}/wasm/transport.cpp
+    ${GIT2CPP_SOURCE_DIR}/wasm/transport.hpp
     ${GIT2CPP_SOURCE_DIR}/wrapper/annotated_commit_wrapper.cpp
     ${GIT2CPP_SOURCE_DIR}/wrapper/annotated_commit_wrapper.hpp
     ${GIT2CPP_SOURCE_DIR}/wrapper/branch_wrapper.cpp

README.md

@@ -11,6 +11,10 @@ used on any POSIX-compliant system.
 
 See `overview.md` for further details.
 
+## Development workflow
+
+### Build
+
 Developer's workflow using `micromamba` to manage the dependencies:
 
 ```bash
@@ -23,19 +27,31 @@ make -j8
 
 The `git2cpp` executable can then be run, e.g. `./git2cpp -v`.
 
+### Test
+
 The CLI is tested using `python`. From the top-level directory:
 
 ```bash
 pytest -v
 ```
 
+Some tests access the private repository at https://github.com/QuantStack/git2cpp-test-private using
+a fine-grained github Personal Access Token (PAT). These tests are skipped by default. To run them
+you will need to obtain the PAT from one of the maintainers, and run the tests as follows:
+
+```bash
+GIT2CPP_TEST_PRIVATE_TOKEN=<this-is-the-personal-access-token> pytest -v
+```
+
+### pre-commit
+
 `pre-commit` runs automatically on `git commit`. To run it manually use:
 
 ```bash
 pre-commit run --all-files
 ```
 
-# WebAssembly build and deployment
+## WebAssembly build and deployment
 
 The `wasm` directory contains everything needed to build the local `git2cpp` source code as an
 WebAssembly [Emscripten-forge](https://emscripten-forge.org/) package, create local
@@ -48,7 +64,7 @@ See the `README.md` in the `wasm` directory for further details.
 The latest `cockle` and JupyterLite `terminal` deployments using `git2cpp` are available at
 [https://quantstack.net/git2cpp](https://quantstack.net/git2cpp)
 
-# Documentation
+## Documentation
 
 The project documentation is generated from the `git2cpp` help pages. To build the documentation
 locally first build `git2cpp` as usual as described above, then install the documentation

src/subcommand/clone_subcommand.cpp

@@ -5,6 +5,7 @@
 #include "../utils/credentials.hpp"
 #include "../utils/input_output.hpp"
 #include "../utils/progress.hpp"
+#include "../wasm/scope.hpp"
 #include "../wrapper/repository_wrapper.hpp"
 
 clone_subcommand::clone_subcommand(const libgit2_object&, CLI::App& app)
@@ -29,6 +30,8 @@ clone_subcommand::clone_subcommand(const libgit2_object&, CLI::App& app)
 
 void clone_subcommand::run()
 {
+    wasm_http_transport_scope transport;  // Enables wasm http(s) transport.
+
     // m_depth = 0 means no shallow clone in libgit2, while
     // it is forbidden with git. Therefore we use another
     // sentinel value to detect full clone.

src/subcommand/fetch_subcommand.cpp

@@ -7,6 +7,7 @@
 #include "../utils/credentials.hpp"
 #include "../utils/input_output.hpp"
 #include "../utils/progress.hpp"
+#include "../wasm/scope.hpp"
 #include "../wrapper/repository_wrapper.hpp"
 
 fetch_subcommand::fetch_subcommand(const libgit2_object&, CLI::App& app)
@@ -32,6 +33,8 @@ fetch_subcommand::fetch_subcommand(const libgit2_object&, CLI::App& app)
 
 void fetch_subcommand::run()
 {
+    wasm_http_transport_scope transport;  // Enables wasm http(s) transport.
+
     auto directory = get_current_git_path();
     auto repo = repository_wrapper::open(directory);
 

src/subcommand/push_subcommand.cpp

@@ -6,6 +6,7 @@
 
 #include "../utils/credentials.hpp"
 #include "../utils/progress.hpp"
+#include "../wasm/scope.hpp"
 #include "../wrapper/repository_wrapper.hpp"
 
 push_subcommand::push_subcommand(const libgit2_object&, CLI::App& app)
@@ -26,6 +27,8 @@ push_subcommand::push_subcommand(const libgit2_object&, CLI::App& app)
 
 void push_subcommand::run()
 {
+    wasm_http_transport_scope transport;  // Enables wasm http(s) transport.
+
     auto directory = get_current_git_path();
     auto repo = repository_wrapper::open(directory);
 

src/utils/common.cpp

@@ -5,6 +5,7 @@
 #include <iostream>
 #include <map>
 #include <ranges>
+#include <regex>
 #include <sstream>
 
 #include <git2.h>
@@ -135,3 +136,9 @@ std::vector<std::string> split_input_at_newlines(std::string_view str)
                  );
     return std::vector<std::string>{split.begin(), split.end()};
 }
+
+std::string trim(const std::string& str)
+{
+    auto s = std::regex_replace(str, std::regex("^\\s+"), "");
+    return std::regex_replace(s, std::regex("\\s+$"), "");
+}

src/utils/common.hpp

@@ -76,3 +76,6 @@ class git_strarray_wrapper
 std::string read_file(const std::string& path);
 
 std::vector<std::string> split_input_at_newlines(std::string_view str);
+
+// Remove whitespace from start and end of a string.
+std::string trim(const std::string& str);

src/utils/credentials.cpp

@@ -25,15 +25,15 @@ int user_credentials(
         std::string username = username_from_url ? username_from_url : "";
         if (username.empty())
         {
-            username = prompt_input("Username: ");
+            username = trim(prompt_input("Username: "));
         }
         if (username.empty())
         {
             giterr_set_str(GIT_ERROR_HTTP, "No username specified");
             return GIT_EAUTH;
         }
 
-        std::string password = prompt_input("Password: ", false);
+        std::string password = trim(prompt_input("Password: ", false));
         if (password.empty())
         {
             giterr_set_str(GIT_ERROR_HTTP, "No password specified");

src/wasm/libgit2_internals.cpp

@@ -0,0 +1,66 @@
+#ifdef EMSCRIPTEN
+
+#    include "libgit2_internals.hpp"
+
+// http method and service.
+
+std::string name_for_method(git_http_method method)
+{
+    switch (method)
+    {
+        case GIT_HTTP_METHOD_GET:
+            return "GET";
+        case GIT_HTTP_METHOD_POST:
+            return "POST";
+        case GIT_HTTP_METHOD_CONNECT:
+            return "CONNECT";
+    }
+    return "";
+}
+
+std::optional<http_service> select_service(git_smart_service_t action)
+{
+    switch (action)
+    {
+        case GIT_SERVICE_UPLOADPACK_LS:
+            return http_service{
+                GIT_HTTP_METHOD_GET,
+                "/info/refs?service=git-upload-pack",
+                nullptr,
+                "application/x-git-upload-pack-advertisement",
+                1,
+                0
+            };
+        case GIT_SERVICE_UPLOADPACK:
+            return http_service{
+                GIT_HTTP_METHOD_POST,
+                "/git-upload-pack",
+                "application/x-git-upload-pack-request",
+                "application/x-git-upload-pack-result",
+                0,
+                0
+            };
+        case GIT_SERVICE_RECEIVEPACK_LS:
+            return http_service{
+                GIT_HTTP_METHOD_GET,
+                "/info/refs?service=git-receive-pack",
+                nullptr,
+                "application/x-git-receive-pack-advertisement",
+                1,
+                0
+            };
+        case GIT_SERVICE_RECEIVEPACK:
+            return http_service{
+                GIT_HTTP_METHOD_POST,
+                "/git-receive-pack",
+                "application/x-git-receive-pack-request",
+                "application/x-git-receive-pack-result",
+                0,
+                1
+            };
+    }
+
+    return std::nullopt;
+}
+
+#endif  // EMSCRIPTEN

src/wasm/libgit2_internals.hpp

@@ -0,0 +1,66 @@
+#pragma once
+
+#ifdef EMSCRIPTEN
+
+#    include <optional>
+#    include <string>
+
+#    include <git2/sys/errors.h>
+#    include <git2/sys/transport.h>
+
+// Libgit2 internals that we want to use so they are reproduced here in some form.
+
+// asserts
+
+#    define GIT_ASSERT(expr) GIT_ASSERT_WITH_RETVAL(expr, -1)
+
+#    define GIT_ASSERT_ARG(expr) GIT_ASSERT_ARG_WITH_RETVAL(expr, -1)
+
+#    define GIT_ASSERT_WITH_RETVAL(expr, fail) \
+        GIT_ASSERT__WITH_RETVAL(expr, 0, "unrecoverable internal error", fail)
+
+#    define GIT_ASSERT_ARG_WITH_RETVAL(expr, fail) GIT_ASSERT__WITH_RETVAL(expr, 0, "invalid argument", fail)
+
+#    define GIT_ASSERT__WITH_RETVAL(expr, code, msg, fail)   \
+        do                                                   \
+        {                                                    \
+            if (!(expr))                                     \
+            {                                                \
+                git_error_set(code, "%s: '%s'", msg, #expr); \
+                return fail;                                 \
+            }                                                \
+        } while (0)
+
+// http status code, method and service.
+
+#    define GIT_HTTP_STATUS_CONTINUE 100
+#    define GIT_HTTP_STATUS_OK 200
+#    define GIT_HTTP_MOVED_PERMANENTLY 301
+#    define GIT_HTTP_FOUND 302
+#    define GIT_HTTP_SEE_OTHER 303
+#    define GIT_HTTP_TEMPORARY_REDIRECT 307
+#    define GIT_HTTP_PERMANENT_REDIRECT 308
+#    define GIT_HTTP_STATUS_UNAUTHORIZED 401
+#    define GIT_HTTP_STATUS_PROXY_AUTHENTICATION_REQUIRED 407
+
+typedef enum
+{
+    GIT_HTTP_METHOD_GET,
+    GIT_HTTP_METHOD_POST,
+    GIT_HTTP_METHOD_CONNECT
+} git_http_method;
+
+typedef struct
+{
+    git_http_method m_method;
+    std::string m_url;
+    std::string m_request_type;
+    std::string m_response_type;
+    unsigned int m_initial : 1, m_chunked : 1;
+} http_service;
+
+std::string name_for_method(git_http_method method);
+
+std::optional<http_service> select_service(git_smart_service_t action);
+
+#endif  // EMSCRIPTEN

src/wasm/response.cpp

@@ -0,0 +1,74 @@
+#ifdef EMSCRIPTEN
+
+#    include "response.hpp"
+
+#    include "../utils/common.hpp"
+#    include "libgit2_internals.hpp"
+
+wasm_http_response::wasm_http_response(char* buffer, size_t buffer_size, size_t* bytes_read)
+    : m_buffer(buffer)
+    , m_buffer_size(buffer_size)
+    , m_bytes_read(bytes_read)
+    , m_status(0)
+{
+    *m_bytes_read = 0;
+}
+
+void wasm_http_response::add_header(const std::string& key, const std::string& value)
+{
+    m_response_headers.emplace(key, trim(value));
+}
+
+void wasm_http_response::clear()
+{
+    *m_bytes_read = 0;
+    m_status = 0;
+    m_status_text.clear();
+    m_response_headers.clear();
+}
+
+std::optional<std::string> wasm_http_response::get_header(const std::string& key) const
+{
+    // Return the first header with the specified key.
+    // If we ever have to handle multiple headers with the same key, will need to do something more
+    // complicated here.
+    auto header = m_response_headers.find(key);
+    if (header != m_response_headers.end())
+    {
+        return header->second;
+    }
+    return std::nullopt;
+}
+
+bool wasm_http_response::has_header(const std::string& key) const
+{
+    return m_response_headers.find(key) != m_response_headers.end();
+}
+
+bool wasm_http_response::has_header_matches(const std::string& key, std::string_view match) const
+{
+    auto range = m_response_headers.equal_range(key);
+    for (auto i = range.first; i != range.second; ++i)
+    {
+        if (i->second == match)
+        {
+            return true;
+        }
+    }
+    return false;
+}
+
+bool wasm_http_response::has_header_starts_with(const std::string& key, std::string_view start) const
+{
+    auto range = m_response_headers.equal_range(key);
+    for (auto i = range.first; i != range.second; ++i)
+    {
+        if (i->second.starts_with(start))
+        {
+            return true;
+        }
+    }
+    return false;
+}
+
+#endif  // EMSCRIPTEN