Skip to content

precompile the v8 compile cache when building #608

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 4 commits into
base: main
Choose a base branch
from
Draft

precompile the v8 compile cache when building #608

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
3535d13
precompile the v8 compile cache when building
rochdev Jan 17, 2025
567df20
improve comment
rochdev Jan 17, 2025
eb2bd8b
fix indentation
rochdev Jan 17, 2025
e2875ab
fix path
rochdev Jan 17, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
75 changes: 39 additions & 36 deletions Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ FROM $image as builder
ARG image

# Create the directory structure required for AWS Lambda Layer
RUN mkdir -p /nodejs/node_modules/
RUN mkdir -p /opt/nodejs/node_modules/

# Install dev dependencies
COPY . datadog-lambda-js
Expand All @@ -12,10 +12,10 @@ RUN yarn install

# Build the lambda layer
RUN yarn build
RUN cp -r dist /nodejs/node_modules/datadog-lambda-js
RUN cp ./src/runtime/module_importer.js /nodejs/node_modules/datadog-lambda-js/runtime
RUN cp -r dist /opt/nodejs/node_modules/datadog-lambda-js
RUN cp ./src/runtime/module_importer.js /opt/nodejs/node_modules/datadog-lambda-js/runtime

RUN cp ./src/handler.mjs /nodejs/node_modules/datadog-lambda-js
RUN cp ./src/handler.mjs /opt/nodejs/node_modules/datadog-lambda-js
RUN rm -rf node_modules

# Move dd-trace from devDependencies to production dependencies
Expand All @@ -25,41 +25,44 @@ RUN mv package-new.json package.json
# Install dependencies
RUN yarn install --production=true
# Copy the dependencies to the modules folder
RUN cp -rf node_modules/* /nodejs/node_modules
RUN cp -rf node_modules/* /opt/nodejs/node_modules

# Remove the AWS SDK, which is installed in the lambda by default
RUN rm -rf /nodejs/node_modules/aws-sdk
RUN rm -rf /nodejs/node_modules/aws-xray-sdk-core/node_modules/aws-sdk
RUN rm -rf /opt/nodejs/node_modules/aws-sdk
RUN rm -rf /opt/nodejs/node_modules/aws-xray-sdk-core/node_modules/aws-sdk

# Remove heavy files from dd-trace which aren't used in a lambda environment
RUN rm -rf /nodejs/node_modules/dd-trace/prebuilds
RUN rm -rf /nodejs/node_modules/dd-trace/dist
RUN rm -rf /nodejs/node_modules/@datadog/libdatadog
RUN rm -rf /nodejs/node_modules/@datadog/native-appsec
RUN rm -rf /nodejs/node_modules/@datadog/native-metrics
RUN rm -rf /nodejs/node_modules/hdr-histogram-js/build
RUN rm -rf /nodejs/node_modules/protobufjs/dist
RUN rm -rf /nodejs/node_modules/protobufjs/cli
RUN rm -rf /nodejs/node_modules/@datadog/pprof/prebuilds/linux-arm
RUN rm -rf /nodejs/node_modules/@datadog/pprof/prebuilds/darwin-arm64
RUN rm -rf /nodejs/node_modules/@datadog/pprof/prebuilds/darwin-x64
RUN rm -rf /nodejs/node_modules/@datadog/pprof/prebuilds/win32-ia32
RUN rm -rf /nodejs/node_modules/@datadog/pprof/prebuilds/win32-x64
RUN rm -rf /nodejs/node_modules/@datadog/native-iast-taint-tracking
RUN rm -rf /nodejs/node_modules/@datadog/native-iast-rewriter
RUN rm -rf /nodejs/node_modules/@datadog/pprof/prebuilds/linuxmusl-x64
RUN rm -rf /nodejs/node_modules/jsonpath-plus/src/jsonpath.d.ts
RUN rm -rf /nodejs/node_modules/jsonpath-plus/src/jsonpath-browser.js
RUN rm -rf /nodejs/node_modules/jsonpath-plus/src/dist/index-browser-umd.min.cjs
RUN rm -rf /nodejs/node_modules/jsonpath-plus/src/dist/index-browser-umd.cjs
RUN rm -rf /nodejs/node_modules/jsonpath-plus/src/dist/index-browser-esm.min.js
RUN rm -rf /nodejs/node_modules/jsonpath-plus/src/dist/index-browser-esm.js
RUN find /nodejs/node_modules -name "*.d.ts" -delete
RUN find /nodejs/node_modules -name "*.js.map" -delete
RUN find /nodejs/node_modules -name "*.mjs.map" -delete
RUN find /nodejs/node_modules -name "*.cjs.map" -delete
RUN find /nodejs/node_modules -name "*.ts.map" -delete
RUN find /nodejs/node_modules -name "*.md" -delete
RUN rm -rf /opt/nodejs/node_modules/dd-trace/prebuilds
RUN rm -rf /opt/nodejs/node_modules/dd-trace/dist
RUN rm -rf /opt/nodejs/node_modules/@datadog/libdatadog
RUN rm -rf /opt/nodejs/node_modules/@datadog/native-appsec
RUN rm -rf /opt/nodejs/node_modules/@datadog/native-metrics
RUN rm -rf /opt/nodejs/node_modules/hdr-histogram-js/build
RUN rm -rf /opt/nodejs/node_modules/protobufjs/dist
RUN rm -rf /opt/nodejs/node_modules/protobufjs/cli
RUN rm -rf /opt/nodejs/node_modules/@datadog/pprof/prebuilds/linux-arm
RUN rm -rf /opt/nodejs/node_modules/@datadog/pprof/prebuilds/darwin-arm64
RUN rm -rf /opt/nodejs/node_modules/@datadog/pprof/prebuilds/darwin-x64
RUN rm -rf /opt/nodejs/node_modules/@datadog/pprof/prebuilds/win32-ia32
RUN rm -rf /opt/nodejs/node_modules/@datadog/pprof/prebuilds/win32-x64
RUN rm -rf /opt/nodejs/node_modules/@datadog/native-iast-taint-tracking
RUN rm -rf /opt/nodejs/node_modules/@datadog/native-iast-rewriter
RUN rm -rf /opt/nodejs/node_modules/@datadog/pprof/prebuilds/linuxmusl-x64
RUN rm -rf /opt/nodejs/node_modules/jsonpath-plus/src/jsonpath.d.ts
RUN rm -rf /opt/nodejs/node_modules/jsonpath-plus/src/jsonpath-browser.js
RUN rm -rf /opt/nodejs/node_modules/jsonpath-plus/src/dist/index-browser-umd.min.cjs
RUN rm -rf /opt/nodejs/node_modules/jsonpath-plus/src/dist/index-browser-umd.cjs
RUN rm -rf /opt/nodejs/node_modules/jsonpath-plus/src/dist/index-browser-esm.min.js
RUN rm -rf /opt/nodejs/node_modules/jsonpath-plus/src/dist/index-browser-esm.js
RUN find /opt/nodejs/node_modules -name "*.d.ts" -delete
RUN find /opt/nodejs/node_modules -name "*.js.map" -delete
RUN find /opt/nodejs/node_modules -name "*.mjs.map" -delete
RUN find /opt/nodejs/node_modules -name "*.cjs.map" -delete
RUN find /opt/nodejs/node_modules -name "*.ts.map" -delete
RUN find /opt/nodejs/node_modules -name "*.md" -delete

# Warm up v8 compile cache
RUN node -e "require('/opt/nodejs/node_modules/datadog-lambda-js/runtime/module_importer').initTracer()"

FROM scratch
COPY --from=builder /nodejs /
COPY --from=builder /opt/nodejs /
3 changes: 2 additions & 1 deletion package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,8 @@
"hot-shots": "8.5.0",
"promise-retry": "^2.0.1",
"serialize-error": "^8.1.0",
"shimmer": "1.2.1"
"shimmer": "1.2.1",
"v8-compile-cache": "^2.4.0"
},
"jest": {
"verbose": true,
Expand Down
21 changes: 21 additions & 0 deletions src/runtime/module_importer.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,34 @@

const { logDebug, updateDDTags } = require("../utils");

function compileCache () {
const { FileSystemBlobStore, NativeCompileCache } = require('v8-compile-cache').__TEST__

const cacheDir = __dirname
const prefix = 'module_importer'
const blobStore = new FileSystemBlobStore(cacheDir, prefix)

const nativeCompileCache = new NativeCompileCache()
nativeCompileCache.setCacheStore(blobStore)
nativeCompileCache.install()

process.once('exit', () => {
if (blobStore.isDirty()) {
blobStore.save()
}
nativeCompileCache.uninstall()
})
}

// Currently no way to prevent typescript from auto-transpiling import into require,
// so we expose a wrapper in js
exports.import = function (path) {
return import(path);
}

exports.initTracer = function () {
compileCache()

// Looks for the function local version of dd-trace first, before using
// the version provided by the layer
const path = require.resolve("dd-trace", { paths: ["/var/task/node_modules", ...module.paths] });
Expand Down
5 changes: 5 additions & 0 deletions yarn.lock
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4075,6 +4075,11 @@ uuid@^9.0.1:
resolved "https://registry.yarnpkg.com/uuid/-/uuid-9.0.1.tgz#e188d4c8853cc722220392c424cd637f32293f30"
integrity sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==

v8-compile-cache@^2.4.0:
version "2.4.0"
resolved "https://registry.yarnpkg.com/v8-compile-cache/-/v8-compile-cache-2.4.0.tgz#cdada8bec61e15865f05d097c5f4fd30e94dc128"
integrity sha512-ocyWc3bAHBB/guyqJQVI5o4BZkPhznPYUG2ea80Gond/BgNWpap8TOmLSeeQG7bnh2KMISxskdADG59j7zruhw==

v8-to-istanbul@^8.1.0:
version "8.1.1"
resolved "https://registry.yarnpkg.com/v8-to-istanbul/-/v8-to-istanbul-8.1.1.tgz#77b752fd3975e31bbcef938f85e9bd1c7a8d60ed"
Expand Down
Loading