feat: support rust via cargo

.appveyor.yml

@@ -73,6 +73,15 @@ for:
     # setup make
     - "choco install make"
 
+    # setup rust
+    - appveyor-retry appveyor DownloadFile https://win.rustup.rs/ -FileName rustup-init.exe
+    - rustup-init.exe -y --default-host x86_64-pc-windows-msvc --default-toolchain stable
+    - set PATH=%PATH%;C:\Users\appveyor\.cargo\bin
+    - set RUST_BACKTRACE=1
+    - rustup target add x86_64-unknown-linux-musl --toolchain stable
+    - rustc -V
+    - cargo -V
+
     # Echo final Path
     - "echo %PATH%"
 
@@ -103,6 +112,14 @@ for:
       - sh: "sudo unzip -d /opt/gradle /tmp/gradle-*.zip"
       - sh: "PATH=/opt/gradle/gradle-5.5/bin:$PATH"
 
+      # Install rust
+      - sh: "curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal"
+      - sh: "PATH=${HOME}/.cargo/bin:$PATH"
+      - sh: "rustup target add x86_64-unknown-linux-musl --toolchain stable"
+      - sh: "sudo apt-get install -y musl-tools"
+      - sh: "rustc -V"
+      - sh: "cargo -V"
+
       # Install black
       - sh: "wget -O /tmp/black https://github.com/python/black/releases/download/19.10b0/black"
       - sh: "chmod +x /tmp/black"

Makefile

@@ -2,10 +2,13 @@ init:
 	LAMBDA_BUILDERS_DEV=1 pip install -e '.[dev]'
 
 test:
-	# Run unit tests
+	# Run unit and functional tests
 	# Fail if coverage falls below 94%
 	LAMBDA_BUILDERS_DEV=1 pytest --cov aws_lambda_builders --cov-report term-missing --cov-fail-under 94 tests/unit tests/functional
 
+unit-test:
+	LAMBDA_BUILDERS_DEV=1 pytest tests/unit
+
 func-test:
 	LAMBDA_BUILDERS_DEV=1 pytest tests/functional
 

README.md

@@ -15,6 +15,7 @@ Lambda Builders currently contains the following workflows
 * Ruby with Bundler
 * Go with Dep
 * Go with Mod
+* Rust with Cargo
 
 Lambda Builders is the brains behind the `sam build` command from [AWS SAM CLI](https://github.com/awslabs/aws-sam-cli)
 

aws_lambda_builders/workflows/__init__.py

@@ -11,3 +11,4 @@
 import aws_lambda_builders.workflows.java_maven
 import aws_lambda_builders.workflows.dotnet_clipackage
 import aws_lambda_builders.workflows.custom_make
+import aws_lambda_builders.workflows.rust_cargo

aws_lambda_builders/workflows/rust_cargo/DESIGN.md

@@ -0,0 +1,44 @@
+# Rust Cargo Builder
+
+## Scope
+
+This package enables the creation of a Lambda deployment package for Rust projects managed using the [cargo](https://doc.rust-lang.org/cargo/) build tool targeting Lambda's "provided" runtime. Rust support for the provided runtime is bundled as a compilation dependency of these projects, provided by the [lambda](https://github.com/awslabs/aws-lambda-rust-runtime) crate.
+
+## Implementation
+
+The general algorithm for preparing a rust executable for use on AWS Lambda
+is as follows.
+
+### Build
+
+It builds a binary in the standard cargo target directory. 
+
+### Copy and Rename executable
+
+It then copies the executable to the target directory renaming the executable to "bootstrap" to honor the provided runtime's [expectation on executable names](https://docs.aws.amazon.com/lambda/latest/dg/runtimes-custom.html).
+
+## Challenges
+
+### Cross compilation
+
+Cargo builds binaries targeting host platforms. When the host platform is not the same as the target platform it leverages a parameterized notion of a named target, typically installed and managed by [rustup](https://github.com/rust-lang/rustup). In the case of `sam` we default this target to to `x86_64-unknown-linux-musl` when the host is not linux based.
+
+Users can simply run `rustup target add x86_64-unknown-linux-musl` if needed. It's also possible for sam to detect this when needed and do that for users by parsing the output of `rustup target list --installed` but it would be unusual for sam to
+install toolchain components on a users behalf.
+
+The challenge is not installing the cargo target. The challenge is ensuring external linker dependencies of that target are present at build time. Setting this up varies from platform to platform. And example for osx can be found [here](https://www.andrew-thorburn.com/cross-compiling-a-simple-rust-web-app/). This may warrant requiring a building inside a docker container that is linux based.
+
+## Notes
+
+Like the go builders, the workflow argument `options.artifact_executable_name`
+interface is used to provide a handler name that resolves to an executable. This
+enables sam support for cargo workspaces allowing for one rust project to have multiple lambdas. Cargo workspaces have a notion of a `package` and `bin`. A `package` can have
+multiple bins but typically `packages` have a 1-to-1 relationship with a default `bin`: `main.rs`. 
+
+The following table defines handler name to package/bin semantics.
+
+| Handler name | Package | Bin |
+|--------------|---------|-----|
+| foo          | foo     | foo |
+| foo.foo      | foo     | foo |
+| foo.bar      | foo     | bar |

aws_lambda_builders/workflows/rust_cargo/__init__.py

@@ -0,0 +1,5 @@
+"""
+Builds Rust Lambda functions using Cargo
+"""
+
+from .workflow import RustCargoWorkflow

aws_lambda_builders/workflows/rust_cargo/actions.py

@@ -0,0 +1,233 @@
+"""
+Rust Cargo build actions
+"""
+
+import os
+import subprocess
+import shutil
+import json
+
+from aws_lambda_builders.workflow import BuildMode
+from aws_lambda_builders.actions import ActionFailedError, BaseAction, Purpose
+
+CARGO_TARGET = "x86_64-unknown-linux-musl"
+
+
+class BuilderError(Exception):
+    MESSAGE = "Builder Failed: {message}"
+
+    def __init__(self, **kwargs):
+        Exception.__init__(self, self.MESSAGE.format(**kwargs))
+
+
+class OSUtils(object):
+    """
+    Wrapper around file system functions, to make it easy to
+    unit test actions in memory
+    """
+
+    def popen(self, command, stdout=None, stderr=None, env=None, cwd=None):
+        return subprocess.Popen(command, stdout=stdout, stderr=stderr, env=env, cwd=cwd)
+
+    def copyfile(self, source, destination):
+        shutil.copyfile(source, destination)
+
+    def makedirs(self, path):
+        if not os.path.exists(path):
+            os.makedirs(path)
+
+
+def parse_handler(handler):
+    """
+    Parse package and binary from handler name
+
+    If the handler contains a period, assume `package.bin_name`
+    otherwise assume common case where bin_name is the same as the package
+    """
+    spec = handler.split(".", 1)
+    if len(spec) == 1:
+        return (spec[0], spec[0])
+    else:
+        return (spec[0], spec[1])
+
+
+class BuildAction(BaseAction):
+    NAME = "CargoBuild"
+    DESCRIPTION = "Building the project using Cargo"
+    PURPOSE = Purpose.COMPILE_SOURCE
+
+    def __init__(self, source_dir, handler, binaries, platform, mode, osutils=OSUtils()):
+        """
+        Build the a rust executable
+
+        :type source_dir: str
+        :param source_dir:
+            Path to a folder containing the source code
+
+        :type handler: str
+        :param handler:
+            Handler name in `package.bin_name` or `bin_name` format
+
+        :type binaries: dict
+        :param binaries:
+            Resolved path dependencies
+
+        :type platform: string
+        :param platform:
+            Platform builder is being run on
+
+        :type mode: str
+        :param mode:
+            Mode the build should produce
+
+        :type osutils: object
+        :param osutils:
+            Optional, External IO utils
+        """
+        self.source_dir = source_dir
+        self.handler = handler
+        self.mode = mode
+        self.binaries = binaries
+        self.platform = platform
+        self.osutils = osutils
+
+    def cargo_metadata(self):
+        p = self.osutils.popen(
+            ["cargo", "metadata", "--no-deps", "--format-version=1"],
+            stderr=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+            env=os.environ.copy(),
+            cwd=self.source_dir,
+        )
+        out, err = p.communicate()
+        if p.returncode != 0:
+            raise BuilderError(message=err.decode("utf8").strip())
+        return json.loads(out)
+
+    def build_command(self, package):
+        cmd = [self.binaries["cargo"].binary_path, "build", "-p", package, "--target", CARGO_TARGET]
+        if self.mode != BuildMode.DEBUG:
+            cmd.append("--release")
+        return cmd
+
+    def resolve_binary(self, cargo_meta):
+        """
+        Interrogate cargo metadata to resolve a handler function
+
+        :type cargo_meta: dict
+        :param cargo_meta:
+            Build metadata emitted by cargo
+        """
+        (package, binary) = parse_handler(self.handler)
+        exists = any(
+            [
+                kind == "bin"
+                for pkg in cargo_meta["packages"]
+                if pkg["name"] == package
+                for target in pkg["targets"]
+                if target["name"] == binary
+                for kind in target["kind"]
+            ]
+        )
+        if not exists:
+            raise BuilderError(message="Cargo project does not contain a {handler} binary".format(handler=self.handler))
+
+        return (package, binary)
+
+    def build_env(self):
+        env = os.environ.copy()
+        if self.platform.lower() == "darwin":
+            # on osx we assume a musl cross compilation
+            # linker installed via `brew install filosottile/musl-cross/musl-cross`
+            # This requires the follow env vars when invoking cargo build
+            env.update(
+                {
+                    "RUSTFLAGS": "{rust_flags} -Clinker=x86_64-linux-musl-gcc".format(
+                        rust_flags=env.get("RUSTFLAGS", "")
+                    ),
+                    "TARGET_CC": "x86_64-linux-musl-gcc",
+                    "CC_x86_64_unknown_linux_musl": "x86_64-linux-musl-gcc",
+                }
+            )
+        if self.platform.lower() == "windows":
+            # on windows we assume a musl cross compilation
+            # linker is available via rusts embedded llvm linker "rust-lld"
+            # but cc is used as the default
+            # source: https://github.com/KodrAus/rust-cross-compile
+            # This requires the follow env vars when invoking cargo build
+            env.update(
+                {
+                    "RUSTFLAGS": "{rust_flags} -Clinker=rust-lld".format(rust_flags=env.get("RUSTFLAGS", "")),
+                    "TARGET_CC": "rust-lld",
+                    "CC_x86_64_unknown_linux_musl": "rust-lld",
+                }
+            )
+        return env
+
+    def execute(self):
+        try:
+            (package, _) = self.resolve_binary(self.cargo_metadata())
+            p = self.osutils.popen(
+                self.build_command(package),
+                stderr=subprocess.PIPE,
+                stdout=subprocess.PIPE,
+                env=self.build_env(),
+                cwd=self.source_dir,
+            )
+            out, err = p.communicate()
+            if p.returncode != 0:
+                raise BuilderError(message=err.decode("utf8").strip())
+            return out.decode("utf8").strip()
+        except Exception as ex:
+            raise ActionFailedError(str(ex))
+
+
+class CopyAndRenameAction(BaseAction):
+    NAME = "CopyAndRename"
+    DESCRIPTION = "Copy executable renaming if needed"
+    PURPOSE = Purpose.COPY_SOURCE
+
+    def __init__(self, source_dir, handler, artifacts_dir, platform, mode, osutils=OSUtils()):
+        """
+        Copy and rename rust executable
+
+        :type source_dir: str
+        :param source_dir:
+            Path to a folder containing the source code
+
+        :type handler: str
+        :param handler:
+            Handler name in `package.bin_name` or `bin_name` format
+
+        :type artifacts_dir: str
+        :param binaries:
+            Path to a folder containing the deployable artifacts
+
+        :type platform: string
+        :param platform:
+            Platform builder is being run on
+
+        :type mode: str
+        :param mode:
+            Mode the build should produce
+
+        :type osutils: object
+        :param osutils:
+            Optional, External IO utils
+        """
+        self.source_dir = source_dir
+        self.handler = handler
+        self.artifacts_dir = artifacts_dir
+        self.platform = platform
+        self.mode = mode
+        self.osutils = osutils
+
+    def binary_path(self):
+        (_, binary) = parse_handler(self.handler)
+        profile = "debug" if self.mode == BuildMode.DEBUG else "release"
+        target = os.path.join(self.source_dir, "target", CARGO_TARGET)
+        return os.path.join(target, profile, binary)
+
+    def execute(self):
+        self.osutils.makedirs(self.artifacts_dir)
+        self.osutils.copyfile(self.binary_path(), os.path.join(self.artifacts_dir, "bootstrap"))

aws_lambda_builders/workflows/rust_cargo/workflow.py

@@ -0,0 +1,37 @@
+"""
+Rust Cargo Workflow
+"""
+import platform
+from aws_lambda_builders.path_resolver import PathResolver
+from aws_lambda_builders.workflow import BaseWorkflow, Capability
+from .actions import BuildAction, CopyAndRenameAction
+
+
+class RustCargoWorkflow(BaseWorkflow):
+
+    NAME = "RustCargoBuilder"
+
+    CAPABILITY = Capability(language="rust", dependency_manager="cargo", application_framework=None)
+
+    SUPPORTED_MANIFESTS = ["Cargo.toml"]
+
+    def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, runtime=None, mode=None, **kwargs):
+        super(RustCargoWorkflow, self).__init__(
+            source_dir, artifacts_dir, scratch_dir, manifest_path, runtime=runtime, **kwargs
+        )
+
+        # we utilize the handler identifier to
+        # select the binary to build
+        options = kwargs.get("options") or {}
+        handler = options.get("artifact_executable_name", None)
+        system_platform = platform.system().lower()
+        self.actions = [
+            BuildAction(source_dir, handler, self.binaries, system_platform, mode),
+            CopyAndRenameAction(source_dir, handler, artifacts_dir, system_platform, mode),
+        ]
+
+    def get_resolvers(self):
+        """
+        specialized path resolver that just returns the list of executable for the runtime on the path.
+        """
+        return [PathResolver(runtime=self.runtime, binary="cargo")]