Skip to content

Commit d0ab84c

Browse files
jacobtreadjacobtread
committed
feat: new ci for building required layers
1 parent a21958d commit d0ab84c

6 files changed

Lines changed: 187 additions & 12 deletions

File tree

‎.github/workflows/build-release.yml‎

Lines changed: 96 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,8 @@ permissions:
1414
contents: write
1515

1616
jobs:
17-
build:
18-
name: Build
17+
build-amd64:
18+
name: Build AMD64
1919
runs-on: ubuntu-latest
2020
steps:
2121
# Checkout the repo for building
@@ -44,29 +44,113 @@ jobs:
4444
# Copy built binary to output directory
4545
- name: Copy binary to output
4646
run: |
47-
cp target/lambda/docbox-http-lambda/bootstrap.zip docbox-http-lambda.zip
48-
cp target/lambda/docbox-presigned-cleanup-lambda/bootstrap.zip docbox-presigned-cleanup-lambda.zip
47+
cp target/lambda/docbox-http-lambda/bootstrap.zip docbox-http-lambda-amd64.zip
48+
cp target/lambda/docbox-presigned-cleanup-lambda/bootstrap.zip docbox-presigned-cleanup-lambda-amd64.zip
4949
cp target/lambda/docbox-upload-completion-lambda/bootstrap.zip docbox-upload-completion-lambda.zip
5050
shell: bash
5151

52+
# Build the docker layers
53+
- name: Build poppler layer
54+
run: ./layers/make-poppler-layer-amd64.sh
55+
56+
# Merge processing lambda with poppler layer
57+
# (Required for processing pdf files)
58+
- name: Merge processing layers
59+
run: |
60+
mkdir /tmp/processing-lambda
61+
unzip docbox-upload-completion-lambda.zip -d /tmp/processing-lambda
62+
unzip poppler-lambda-layer.zip -d /tmp/processing-lambda
63+
zip -r docbox-upload-completion-lambda-amd64.zip /tmp/processing-lambda
64+
mv /tmp/docbox-upload-completion-lambda-amd64.zip
65+
shell: bash
66+
5267
# Upload the built artifacts
5368
- name: Upload docbox-http-lambda.zip
5469
uses: actions/upload-artifact@v4
5570
with:
56-
name: docbox-http-lambda.zip
57-
path: docbox-http-lambda.zip
71+
name: docbox-http-lambda-amd64.zip
72+
path: docbox-http-lambda-amd64.zip
73+
74+
- name: Upload docbox-presigned-cleanup-lambda-amd64.zip
75+
uses: actions/upload-artifact@v4
76+
with:
77+
name: docbox-presigned-cleanup-lambda-amd64.zip
78+
path: docbox-presigned-cleanup-lambda-amd64.zip
79+
80+
- name: Upload docbox-upload-completion-lambda-amd64.zip
81+
uses: actions/upload-artifact@v4
82+
with:
83+
name: docbox-upload-completion-lambda-amd64.zip
84+
path: docbox-upload-completion-lambda-amd64.zip
85+
86+
build:
87+
name: Build ARM64
88+
runs-on: ubuntu-latest
89+
steps:
90+
# Checkout the repo for building
91+
- uses: actions/checkout@v4
92+
93+
# Setup rust for building the service
94+
- name: Set up Rust
95+
uses: dtolnay/rust-toolchain@stable
96+
with:
97+
toolchain: stable
98+
targets: ${{ matrix.target }}
99+
override: true
100+
101+
# Install zig for cargo lambda
102+
- name: Install zig
103+
run: pip3 install ziglang
104+
105+
# Install cargo lambda
106+
- name: Install cargo-lambda
107+
run: curl -fsSL https://cargo-lambda.info/install.sh | sh
108+
109+
# Compile the lambda binaries
110+
- name: Build lambda binaries
111+
run: cargo lambda build --output-format zip --arm64 --release
112+
113+
# Copy built binary to output directory
114+
- name: Copy binary to output
115+
run: |
116+
cp target/lambda/docbox-http-lambda/bootstrap.zip docbox-http-lambda-arm64.zip
117+
cp target/lambda/docbox-presigned-cleanup-lambda/bootstrap.zip docbox-presigned-cleanup-lambda-arm64.zip
118+
cp target/lambda/docbox-upload-completion-lambda/bootstrap.zip docbox-upload-completion-lambda.zip
119+
shell: bash
120+
121+
# Build the docker layers
122+
- name: Build poppler layer
123+
run: ./layers/make-poppler-layer-arm64sh
124+
125+
# Merge processing lambda with poppler layer
126+
# (Required for processing pdf files)
127+
- name: Merge processing layers
128+
run: |
129+
mkdir /tmp/processing-lambda
130+
unzip docbox-upload-completion-lambda.zip -d /tmp/processing-lambda
131+
unzip poppler-lambda-layer.zip -d /tmp/processing-lambda
132+
zip -r docbox-upload-completion-lambda-arm64.zip /tmp/processing-lambda
133+
mv /tmp/docbox-upload-completion-lambda-arm64.zip
134+
shell: bash
135+
136+
# Upload the built artifacts
137+
- name: Upload docbox-http-lambda-arm64.zip
138+
uses: actions/upload-artifact@v4
139+
with:
140+
name: docbox-http-lambda-arm64.zip
141+
path: docbox-http-lambda-arm64.zip
58142

59-
- name: Upload docbox-presigned-cleanup-lambda.zip
143+
- name: Upload docbox-presigned-cleanup-lambda-arm64.zip
60144
uses: actions/upload-artifact@v4
61145
with:
62-
name: docbox-presigned-cleanup-lambda.zip
63-
path: docbox-presigned-cleanup-lambda.zip
146+
name: docbox-presigned-cleanup-lambda-arm64.zip
147+
path: docbox-presigned-cleanup-lambda-arm64.zip
64148

65-
- name: Upload docbox-upload-completion-lambda.zip
149+
- name: Upload docbox-upload-completion-lambda-arm64.zip
66150
uses: actions/upload-artifact@v4
67151
with:
68-
name: docbox-upload-completion-lambda.zip
69-
path: docbox-upload-completion-lambda.zip
152+
name: docbox-upload-completion-lambda-arm64.zip
153+
path: docbox-upload-completion-lambda-arm64.zip
70154

71155
release:
72156
name: Create Release

‎.gitignore‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,18 @@
11
/target
22
.env
3+
4+
# Terraform local metadata directories
5+
.terraform/*
6+
7+
# Terraform state
8+
*.tfstate
9+
*.tfstate.*
10+
11+
# Terraform variable files
12+
*.tfvars
13+
14+
# Terraform backend config files
15+
*.tfbackend
16+
17+
18+
/terraform

‎layers/copy-dep.sh‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
#!/usr/bin/env bash
2+
3+
set -euo pipefail
4+
5+
BINARY="$1"
6+
DEST="$2"
7+
8+
# Create target directories
9+
mkdir -p "$DEST/bin"
10+
mkdir -p "$DEST/lib"
11+
12+
# Copy the binary itself
13+
cp -v "$BINARY" "$DEST/bin/"
14+
15+
# Use ldd to find shared libraries
16+
ldd "$BINARY" | awk '{
17+
for(i=1;i<=NF;i++){
18+
if($i ~ /^\//){ print $i }
19+
}
20+
}' | sort -u | while read -r lib; do
21+
if [[ -f "$lib" ]]; then
22+
cp -v -n "$lib" "$DEST/lib/"
23+
fi
24+
done

‎layers/make-poppler-layer-amd64.sh‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
#!/usr/bin/env bash
2+
3+
# Build the container
4+
docker buildx build --platform linux/amd64 -f poppler.Dockerfile -t poppler-lambda-layer-amd64 .
5+
6+
# Run a container and copy out the zip then delete it
7+
CONTAINER_ID=$(docker create --platform linux/amd64 poppler-lambda-layer-amd64)
8+
docker cp $CONTAINER_ID:/poppler-lambda-layer.zip ./poppler-lambda-layer.zip
9+
docker rm $CONTAINER_ID

‎layers/make-poppler-layer-arm64.sh‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
#!/usr/bin/env bash
2+
3+
# Build the container
4+
docker buildx build --platform linux/arm64 -f poppler.Dockerfile -t poppler-lambda-layer-arm64.
5+
6+
# Run a container and copy out the zip then delete it
7+
CONTAINER_ID=$(docker create --platform linux/arm64 poppler-lambda-layer-arm64)
8+
docker cp $CONTAINER_ID:/poppler-lambda-layer.zip ./poppler-lambda-layer.zip
9+
docker rm $CONTAINER_ID

‎layers/poppler.Dockerfile‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
# Docker for building the layer for poppler support in AWS lambda
2+
FROM public.ecr.aws/lambda/provided:al2023 AS builder
3+
4+
# Install required dependencies
5+
RUN dnf -y update && \
6+
dnf -y install poppler poppler-utils poppler-data && \
7+
dnf clean all
8+
9+
# Copy bundling helper script
10+
COPY ./copy-dep.sh /tmp/copy-dep.sh
11+
12+
# Create output bundle directory
13+
RUN mkdir -p /bundle/opt/bin /bundle/opt/lib /bundle/opt/fonts /bundle/opt/fontconfig /bundle/opt/poppler-data
14+
15+
# Copy required cli tools and their dependencies
16+
RUN chmod +x /tmp/copy-dep.sh && \
17+
# pdfinfo - Get details about a pdf file
18+
/tmp/copy-dep.sh /usr/bin/pdfinfo /bundle/opt && \
19+
# pdftotext - PDF text extraction
20+
/tmp/copy-dep.sh /usr/bin/pdftotext /bundle/opt && \
21+
# pdftocairo - PDF rendering
22+
/tmp/copy-dep.sh /usr/bin/pdftocairo /bundle/opt
23+
24+
# Copy fonts and Poppler data
25+
RUN cp -r /usr/share/fonts/* /bundle/opt/fonts/ && \
26+
cp -r /etc/fonts/* /bundle/opt/etc/fonts && \
27+
cp -r /usr/share/poppler/* /bundle/opt/share/poppler-data/
28+
29+
# Set permissions
30+
RUN chmod -R 755 /bundle/opt
31+
32+
# Zip up the bundle as the layer
33+
RUN zip /bundle /poppler-lambda-layer.zip

0 commit comments

Comments
 (0)