[llvm] r313359 - Added optional validation of svn sources to Dockerfiles.
Ilya Biryukov via llvm-commits
llvm-commits at lists.llvm.org
Fri Sep 15 06:35:54 PDT 2017
Author: ibiryukov
Date: Fri Sep 15 06:35:54 2017
New Revision: 313359
URL: http://llvm.org/viewvc/llvm-project?rev=313359&view=rev
Log:
Added optional validation of svn sources to Dockerfiles.
Summary: This commit also adds a script to compute sha256 hashes of llvm checkouts.
Reviewers: klimek, mehdi_amini
Reviewed By: klimek
Subscribers: llvm-commits
Differential Revision: https://reviews.llvm.org/D37099
Added:
llvm/trunk/utils/docker/scripts/llvm_checksum/
llvm/trunk/utils/docker/scripts/llvm_checksum/llvm_checksum.py (with props)
llvm/trunk/utils/docker/scripts/llvm_checksum/project_tree.py
Modified:
llvm/trunk/utils/docker/build_docker_image.sh
llvm/trunk/utils/docker/debian8/build/Dockerfile
llvm/trunk/utils/docker/example/build/Dockerfile
llvm/trunk/utils/docker/nvidia-cuda/build/Dockerfile
llvm/trunk/utils/docker/scripts/build_install_llvm.sh
Modified: llvm/trunk/utils/docker/build_docker_image.sh
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/utils/docker/build_docker_image.sh?rev=313359&r1=313358&r2=313359&view=diff
==============================================================================
--- llvm/trunk/utils/docker/build_docker_image.sh (original)
+++ llvm/trunk/utils/docker/build_docker_image.sh Fri Sep 15 06:35:54 2017
@@ -38,6 +38,9 @@ Available options:
Can be specified multiple times.
-i|--install-target name of a cmake install target to build and include in
the resulting archive. Can be specified multiple times.
+ -c|--checksums name of a file, containing checksums of llvm checkout.
+ Script will fail if checksums of the checkout do not
+ match.
Required options: --source and --docker-repository, at least one
--install-target.
@@ -66,6 +69,7 @@ $ ./build_docker_image.sh -s debian8 -d
EOF
}
+CHECKSUMS_FILE=""
SEEN_INSTALL_TARGET=0
while [[ $# -gt 0 ]]; do
case "$1" in
@@ -95,6 +99,11 @@ while [[ $# -gt 0 ]]; do
BUILDSCRIPT_ARGS="$BUILDSCRIPT_ARGS $1 $2"
shift 2
;;
+ -c|--checksums)
+ shift
+ CHECKSUMS_FILE="$1"
+ shift
+ ;;
--)
shift
BUILDSCRIPT_ARGS="$BUILDSCRIPT_ARGS -- $*"
@@ -141,6 +150,11 @@ echo "Using a temporary directory for th
cp -r "$SOURCE_DIR/$IMAGE_SOURCE" "$BUILD_DIR/$IMAGE_SOURCE"
cp -r "$SOURCE_DIR/scripts" "$BUILD_DIR/scripts"
+mkdir "$BUILD_DIR/checksums"
+if [ "$CHECKSUMS_FILE" != "" ]; then
+ cp "$CHECKSUMS_FILE" "$BUILD_DIR/checksums/checksums.txt"
+fi
+
if [ "$DOCKER_TAG" != "" ]; then
DOCKER_TAG=":$DOCKER_TAG"
fi
Modified: llvm/trunk/utils/docker/debian8/build/Dockerfile
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/utils/docker/debian8/build/Dockerfile?rev=313359&r1=313358&r2=313359&view=diff
==============================================================================
--- llvm/trunk/utils/docker/debian8/build/Dockerfile (original)
+++ llvm/trunk/utils/docker/debian8/build/Dockerfile Fri Sep 15 06:35:54 2017
@@ -19,7 +19,7 @@ RUN grep deb /etc/apt/sources.list | \
# Install compiler, python and subversion.
RUN apt-get update && \
apt-get install -y --no-install-recommends ca-certificates gnupg \
- build-essential python2.7 wget subversion ninja-build && \
+ build-essential python wget subversion ninja-build && \
rm -rf /var/lib/apt/lists/*
# Import public key required for verifying signature of cmake download.
@@ -37,9 +37,11 @@ RUN mkdir /tmp/cmake-install && cd /tmp/
tar xzf cmake-3.7.2-Linux-x86_64.tar.gz -C /usr/local --strip-components=1 && \
cd / && rm -rf /tmp/cmake-install
+ADD checksums /tmp/checksums
+ADD scripts /tmp/scripts
+
# Arguments passed to build_install_clang.sh.
ARG buildscript_args
# Run the build. Results of the build will be available as /tmp/clang.tar.gz.
-ADD scripts/build_install_llvm.sh /tmp
-RUN /tmp/build_install_llvm.sh ${buildscript_args}
+RUN /tmp/scripts/build_install_llvm.sh ${buildscript_args}
Modified: llvm/trunk/utils/docker/example/build/Dockerfile
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/utils/docker/example/build/Dockerfile?rev=313359&r1=313358&r2=313359&view=diff
==============================================================================
--- llvm/trunk/utils/docker/example/build/Dockerfile (original)
+++ llvm/trunk/utils/docker/example/build/Dockerfile Fri Sep 15 06:35:54 2017
@@ -18,9 +18,11 @@ LABEL maintainer "Maintainer <maintainer
# FIXME: Install llvm/clang build dependencies. Including compiler to
# build stage1, cmake, subversion, ninja, etc.
-# Arguments to pass to build_install_clang.sh.
+ADD checksums /tmp/checksums
+ADD scripts /tmp/scripts
+
+# Arguments passed to build_install_clang.sh.
ARG buildscript_args
# Run the build. Results of the build will be available as /tmp/clang.tar.gz.
-ADD scripts/build_install_llvm.sh /tmp
-RUN /tmp/build_install_llvm.sh ${buildscript_args}
+RUN /tmp/scripts/build_install_llvm.sh ${buildscript_args}
Modified: llvm/trunk/utils/docker/nvidia-cuda/build/Dockerfile
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/utils/docker/nvidia-cuda/build/Dockerfile?rev=313359&r1=313358&r2=313359&view=diff
==============================================================================
--- llvm/trunk/utils/docker/nvidia-cuda/build/Dockerfile (original)
+++ llvm/trunk/utils/docker/nvidia-cuda/build/Dockerfile Fri Sep 15 06:35:54 2017
@@ -17,10 +17,15 @@ ARG buildscript_args
# Install llvm build dependencies.
RUN apt-get update && \
- apt-get install -y --no-install-recommends ca-certificates cmake python2.7 \
+ apt-get install -y --no-install-recommends ca-certificates cmake python \
subversion ninja-build && \
rm -rf /var/lib/apt/lists/*
+ADD checksums /tmp/checksums
+ADD scripts /tmp/scripts
+
+# Arguments passed to build_install_clang.sh.
+ARG buildscript_args
+
# Run the build. Results of the build will be available as /tmp/clang.tar.gz.
-ADD scripts/build_install_llvm.sh /tmp
-RUN /tmp/build_install_llvm.sh ${buildscript_args}
+RUN /tmp/scripts/build_install_llvm.sh ${buildscript_args}
Modified: llvm/trunk/utils/docker/scripts/build_install_llvm.sh
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/utils/docker/scripts/build_install_llvm.sh?rev=313359&r1=313358&r2=313359&view=diff
==============================================================================
--- llvm/trunk/utils/docker/scripts/build_install_llvm.sh (original)
+++ llvm/trunk/utils/docker/scripts/build_install_llvm.sh Fri Sep 15 06:35:54 2017
@@ -181,6 +181,16 @@ if [ $CLANG_TOOLS_EXTRA_ENABLED -ne 0 ];
"$CLANG_BUILD_DIR/src/clang/tools/extra"
fi
+CHECKSUMS_FILE="/tmp/checksums/checksums.txt"
+
+if [ -f "$CHECKSUMS_FILE" ]; then
+ echo "Validating checksums for LLVM checkout..."
+ python "$(dirname $0)/llvm_checksum/llvm_checksum.py" -c "$CHECKSUMS_FILE" \
+ --partial --multi_dir "$CLANG_BUILD_DIR/src"
+else
+ echo "Skipping checksumming checks..."
+fi
+
mkdir "$CLANG_BUILD_DIR/build"
pushd "$CLANG_BUILD_DIR/build"
Added: llvm/trunk/utils/docker/scripts/llvm_checksum/llvm_checksum.py
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/utils/docker/scripts/llvm_checksum/llvm_checksum.py?rev=313359&view=auto
==============================================================================
--- llvm/trunk/utils/docker/scripts/llvm_checksum/llvm_checksum.py (added)
+++ llvm/trunk/utils/docker/scripts/llvm_checksum/llvm_checksum.py Fri Sep 15 06:35:54 2017
@@ -0,0 +1,198 @@
+#!/usr/bin/python
+""" A small program to compute checksums of LLVM checkout.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import hashlib
+import logging
+import re
+import sys
+from argparse import ArgumentParser
+from project_tree import *
+
+SVN_DATES_REGEX = re.compile(r"\$(Date|LastChangedDate)[^\$]+\$")
+
+
+def main():
+ parser = ArgumentParser()
+ parser.add_argument(
+ "-v", "--verbose", action="store_true", help="enable debug logging")
+ parser.add_argument(
+ "-c",
+ "--check",
+ metavar="reference_file",
+ help="read checksums from reference_file and " +
+ "check they match checksums of llvm_path.")
+ parser.add_argument(
+ "--partial",
+ action="store_true",
+ help="ignore projects from reference_file " +
+ "that are not checked out in llvm_path.")
+ parser.add_argument(
+ "--multi_dir",
+ action="store_true",
+ help="indicates llvm_path contains llvm, checked out " +
+ "into multiple directories, as opposed to a " +
+ "typical single source tree checkout.")
+ parser.add_argument("llvm_path")
+
+ args = parser.parse_args()
+ if args.check is not None:
+ with open(args.check, "r") as f:
+ reference_checksums = ReadLLVMChecksums(f)
+ else:
+ reference_checksums = None
+
+ if args.verbose:
+ logging.basicConfig(level=logging.DEBUG)
+
+ llvm_projects = CreateLLVMProjects(not args.multi_dir)
+ checksums = ComputeLLVMChecksums(args.llvm_path, llvm_projects)
+
+ if reference_checksums is None:
+ WriteLLVMChecksums(checksums, sys.stdout)
+ sys.exit(0)
+
+ if not ValidateChecksums(reference_checksums, checksums, args.partial):
+ sys.stdout.write("Checksums differ.\nNew checksums:\n")
+ WriteLLVMChecksums(checksums, sys.stdout)
+ sys.stdout.write("Reference checksums:\n")
+ WriteLLVMChecksums(reference_checksums, sys.stdout)
+ sys.exit(1)
+ else:
+ sys.stdout.write("Checksums match.")
+
+
+def ComputeLLVMChecksums(root_path, projects):
+ """Compute checksums for LLVM sources checked out using svn.
+
+ Args:
+ root_path: a directory of llvm checkout.
+ projects: a list of LLVMProject instances, which describe checkout paths,
+ relative to root_path.
+
+ Returns:
+ A dict mapping from project name to project checksum.
+ """
+ hash_algo = hashlib.sha256
+
+ def collapse_svn_substitutions(contents):
+ # Replace svn substitutions for $Date$ and $LastChangedDate$.
+ # Unfortunately, these are locale-specific.
+ return SVN_DATES_REGEX.sub("$\1$", contents)
+
+ def read_and_collapse_svn_subsitutions(file_path):
+ with open(file_path, "rb") as f:
+ contents = f.read()
+ new_contents = collapse_svn_substitutions(contents)
+ if contents != new_contents:
+ logging.debug("Replaced svn keyword substitutions in %s", file_path)
+ logging.debug("\n\tBefore\n%s\n\tAfter\n%s", contents, new_contents)
+ return new_contents
+
+ project_checksums = dict()
+ # Hash each project.
+ for proj in projects:
+ project_root = os.path.join(root_path, proj.relpath)
+ if not os.path.exists(project_root):
+ logging.info("Folder %s doesn't exist, skipping project %s", proj.relpath,
+ proj.name)
+ continue
+
+ files = list()
+
+ def add_file_hash(file_path):
+ if os.path.islink(file_path) and not os.path.exists(file_path):
+ content = os.readlink(file_path)
+ else:
+ content = read_and_collapse_svn_subsitutions(file_path)
+ hasher = hash_algo()
+ hasher.update(content)
+ file_digest = hasher.hexdigest()
+ logging.debug("Checksum %s for file %s", file_digest, file_path)
+ files.append((file_path, file_digest))
+
+ logging.info("Computing checksum for %s", proj.name)
+ WalkProjectFiles(root_path, projects, proj, add_file_hash)
+
+ # Compute final checksum.
+ files.sort(key=lambda x: x[0])
+ hasher = hash_algo()
+ for file_path, file_digest in files:
+ file_path = os.path.relpath(file_path, project_root)
+ hasher.update(file_path)
+ hasher.update(file_digest)
+ project_checksums[proj.name] = hasher.hexdigest()
+ return project_checksums
+
+
+def WriteLLVMChecksums(checksums, f):
+ """Writes checksums to a text file.
+
+ Args:
+ checksums: a dict mapping from project name to project checksum (result of
+ ComputeLLVMChecksums).
+ f: a file object to write into.
+ """
+
+ for proj in sorted(checksums.keys()):
+ f.write("{} {}\n".format(checksums[proj], proj))
+
+
+def ReadLLVMChecksums(f):
+ """Reads checksums from a text file, produced by WriteLLVMChecksums.
+
+ Returns:
+ A dict, mapping from project name to project checksum.
+ """
+ checksums = {}
+ while True:
+ line = f.readline()
+ if line == "":
+ break
+ checksum, proj = line.split()
+ checksums[proj] = checksum
+ return checksums
+
+
+def ValidateChecksums(reference_checksums,
+ new_checksums,
+ allow_missing_projects=False):
+ """Validates that reference_checksums and new_checksums match.
+
+ Args:
+ reference_checksums: a dict of reference checksums, mapping from a project
+ name to a project checksum.
+ new_checksums: a dict of checksums to be checked, mapping from a project
+ name to a project checksum.
+ allow_missing_projects:
+ When True, reference_checksums may contain more projects than
+ new_checksums. Projects missing from new_checksums are ignored.
+ When False, new_checksums and reference_checksums must contain checksums
+ for the same set of projects. If there is a project in
+ reference_checksums, missing from new_checksums, ValidateChecksums
+ will return False.
+
+ Returns:
+ True, if checksums match with regards to allow_missing_projects flag value.
+ False, otherwise.
+ """
+ if not allow_missing_projects:
+ if len(new_checksums) != len(reference_checksums):
+ return False
+
+ for proj, checksum in new_checksums.iteritems():
+ # We never computed a checksum for this project.
+ if proj not in reference_checksums:
+ return False
+ # Checksum did not match.
+ if reference_checksums[proj] != checksum:
+ return False
+
+ return True
+
+
+if __name__ == "__main__":
+ main()
Propchange: llvm/trunk/utils/docker/scripts/llvm_checksum/llvm_checksum.py
------------------------------------------------------------------------------
svn:executable = *
Added: llvm/trunk/utils/docker/scripts/llvm_checksum/project_tree.py
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/utils/docker/scripts/llvm_checksum/project_tree.py?rev=313359&view=auto
==============================================================================
--- llvm/trunk/utils/docker/scripts/llvm_checksum/project_tree.py (added)
+++ llvm/trunk/utils/docker/scripts/llvm_checksum/project_tree.py Fri Sep 15 06:35:54 2017
@@ -0,0 +1,95 @@
+"""Contains helper functions to compute checksums for LLVM checkouts.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import logging
+import os
+import os.path
+import sys
+
+
+class LLVMProject(object):
+ """An LLVM project with a descriptive name and a relative checkout path.
+ """
+
+ def __init__(self, name, relpath):
+ self.name = name
+ self.relpath = relpath
+
+ def is_subproject(self, other_project):
+ """ Check if self is checked out as a subdirectory of other_project.
+ """
+ return self.relpath.startswith(other_project.relpath)
+
+
+def WalkProjectFiles(checkout_root, all_projects, project, visitor):
+ """ Walk over all files inside a project without recursing into subprojects, '.git' and '.svn' subfolders.
+
+ checkout_root: root of the LLVM checkout.
+ all_projects: projects in the LLVM checkout.
+ project: a project to walk the files of. Must be inside all_projects.
+ visitor: a function called on each visited file.
+ """
+ assert project in all_projects
+
+ ignored_paths = set()
+ for other_project in all_projects:
+ if other_project != project and other_project.is_subproject(project):
+ ignored_paths.add(os.path.join(checkout_root, other_project.relpath))
+
+ def raise_error(err):
+ raise err
+
+ project_root = os.path.join(checkout_root, project.relpath)
+ for root, dirs, files in os.walk(project_root, onerror=raise_error):
+ dirs[:] = [
+ d for d in dirs
+ if d != ".svn" and d != ".git" and
+ os.path.join(root, d) not in ignored_paths
+ ]
+ for f in files:
+ visitor(os.path.join(root, f))
+
+
+def CreateLLVMProjects(single_tree_checkout):
+ """Returns a list of LLVMProject instances, describing relative paths of a typical LLVM checkout.
+
+ Args:
+ single_tree_checkout:
+ When True, relative paths for each project points to a typical single
+ source tree checkout.
+ When False, relative paths for each projects points to a separate
+ directory. However, clang-tools-extra is an exception, its relative path
+ will always be 'clang/tools/extra'.
+ """
+ # FIXME: cover all of llvm projects.
+
+ # Projects that reside inside 'projects/' in a single source tree checkout.
+ ORDINARY_PROJECTS = [
+ "compiler-rt", "dragonegg", "libcxx", "libcxxabi", "libunwind",
+ "parallel-libs", "test-suite"
+ ]
+ # Projects that reside inside 'tools/' in a single source tree checkout.
+ TOOLS_PROJECTS = ["clang", "lld", "lldb", "llgo"]
+
+ if single_tree_checkout:
+ projects = [LLVMProject("llvm", "")]
+ projects += [
+ LLVMProject(p, os.path.join("projects", p)) for p in ORDINARY_PROJECTS
+ ]
+ projects += [
+ LLVMProject(p, os.path.join("tools", p)) for p in TOOLS_PROJECTS
+ ]
+ projects.append(
+ LLVMProject("clang-tools-extra",
+ os.path.join("tools", "clang", "tools", "extra")))
+ else:
+ projects = [LLVMProject("llvm", "llvm")]
+ projects += [LLVMProject(p, p) for p in ORDINARY_PROJECTS]
+ projects += [LLVMProject(p, p) for p in TOOLS_PROJECTS]
+ projects.append(
+ LLVMProject("clang-tools-extra", os.path.join("clang", "tools",
+ "extra")))
+ return projects
More information about the llvm-commits
mailing list