Added optional validation of svn sources to Dockerfiles.

Summary: This commit also adds a script to compute sha256 hashes of llvm checkouts.

Reviewers: klimek, mehdi_amini

Reviewed By: klimek

Subscribers: llvm-commits

Differential Revision: https://reviews.llvm.org/D37099

llvm-svn: 313359

utils/docker/build_docker_image.sh

@@ -38,6 +38,9 @@ Available options:
                         Can be specified multiple times.
     -i|--install-target name of a cmake install target to build and include in
                         the resulting archive. Can be specified multiple times.
+    -c|--checksums      name of a file, containing checksums of llvm checkout.
+                        Script will fail if checksums of the checkout do not
+                        match.
 
 Required options: --source and --docker-repository, at least one
   --install-target.
@@ -66,6 +69,7 @@ $ ./build_docker_image.sh -s debian8 -d mydocker/clang-debian8 -t "latest" \
 EOF
 }
 
+CHECKSUMS_FILE=""
 SEEN_INSTALL_TARGET=0
 while [[ $# -gt 0 ]]; do
   case "$1" in
@@ -95,6 +99,11 @@ while [[ $# -gt 0 ]]; do
       BUILDSCRIPT_ARGS="$BUILDSCRIPT_ARGS $1 $2"
       shift 2
       ;;
+    -c|--checksums)
+      shift
+      CHECKSUMS_FILE="$1"
+      shift
+      ;;
     --)
       shift
       BUILDSCRIPT_ARGS="$BUILDSCRIPT_ARGS -- $*"
@@ -141,6 +150,11 @@ echo "Using a temporary directory for the build: $BUILD_DIR"
 cp -r "$SOURCE_DIR/$IMAGE_SOURCE" "$BUILD_DIR/$IMAGE_SOURCE"
 cp -r "$SOURCE_DIR/scripts" "$BUILD_DIR/scripts"
 
+mkdir "$BUILD_DIR/checksums"
+if [ "$CHECKSUMS_FILE" != "" ]; then
+  cp "$CHECKSUMS_FILE" "$BUILD_DIR/checksums/checksums.txt"
+fi
+
 if [ "$DOCKER_TAG" != "" ]; then
   DOCKER_TAG=":$DOCKER_TAG"
 fi

utils/docker/debian8/build/Dockerfile

@@ -19,7 +19,7 @@ RUN grep deb /etc/apt/sources.list | \
 # Install compiler, python and subversion.
 RUN apt-get update && \
     apt-get install -y --no-install-recommends ca-certificates gnupg \
-    	    build-essential python2.7 wget subversion ninja-build && \
+           build-essential python wget subversion ninja-build && \
     rm -rf /var/lib/apt/lists/*
 
 # Import public key required for verifying signature of cmake download.
@@ -37,9 +37,11 @@ RUN mkdir /tmp/cmake-install && cd /tmp/cmake-install && \
     tar xzf cmake-3.7.2-Linux-x86_64.tar.gz -C /usr/local --strip-components=1 && \
     cd / && rm -rf /tmp/cmake-install
 
+ADD checksums /tmp/checksums
+ADD scripts /tmp/scripts
+
 # Arguments passed to build_install_clang.sh.
 ARG buildscript_args
 
 # Run the build. Results of the build will be available as /tmp/clang.tar.gz.
-ADD scripts/build_install_llvm.sh /tmp
-RUN /tmp/build_install_llvm.sh ${buildscript_args}
+RUN /tmp/scripts/build_install_llvm.sh ${buildscript_args}

utils/docker/example/build/Dockerfile

@@ -18,9 +18,11 @@ LABEL maintainer "Maintainer <maintainer@email>"
 # FIXME: Install llvm/clang build dependencies. Including compiler to
 # build stage1, cmake, subversion, ninja, etc.
 
-# Arguments to pass to build_install_clang.sh.
+ADD checksums /tmp/checksums
+ADD scripts /tmp/scripts
+
+# Arguments passed to build_install_clang.sh.
 ARG buildscript_args
 
 # Run the build. Results of the build will be available as /tmp/clang.tar.gz.
-ADD scripts/build_install_llvm.sh /tmp
-RUN /tmp/build_install_llvm.sh ${buildscript_args}
+RUN /tmp/scripts/build_install_llvm.sh ${buildscript_args}

utils/docker/nvidia-cuda/build/Dockerfile

@@ -17,10 +17,15 @@ ARG buildscript_args
 
 # Install llvm build dependencies.
 RUN apt-get update && \
-    apt-get install -y --no-install-recommends ca-certificates cmake python2.7 \
+    apt-get install -y --no-install-recommends ca-certificates cmake python \
 		    subversion ninja-build && \
     rm -rf /var/lib/apt/lists/*
 
+ADD checksums /tmp/checksums
+ADD scripts /tmp/scripts
+
+# Arguments passed to build_install_clang.sh.
+ARG buildscript_args
+
 # Run the build. Results of the build will be available as /tmp/clang.tar.gz.
-ADD scripts/build_install_llvm.sh /tmp
-RUN /tmp/build_install_llvm.sh ${buildscript_args}
+RUN /tmp/scripts/build_install_llvm.sh ${buildscript_args}

utils/docker/scripts/build_install_llvm.sh

@@ -181,6 +181,16 @@ if [ $CLANG_TOOLS_EXTRA_ENABLED -ne 0 ]; then
     "$CLANG_BUILD_DIR/src/clang/tools/extra"
 fi
 
+CHECKSUMS_FILE="/tmp/checksums/checksums.txt"
+
+if [ -f "$CHECKSUMS_FILE" ]; then
+  echo "Validating checksums for LLVM checkout..."
+  python "$(dirname $0)/llvm_checksum/llvm_checksum.py" -c "$CHECKSUMS_FILE" \
+    --partial --multi_dir "$CLANG_BUILD_DIR/src"
+else
+  echo "Skipping checksumming checks..."
+fi
+
 mkdir "$CLANG_BUILD_DIR/build"
 pushd "$CLANG_BUILD_DIR/build"
 

utils/docker/scripts/llvm_checksum/llvm_checksum.py

@@ -0,0 +1,198 @@
+#!/usr/bin/python
+""" A small program to compute checksums of LLVM checkout.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import hashlib
+import logging
+import re
+import sys
+from argparse import ArgumentParser
+from project_tree import *
+
+SVN_DATES_REGEX = re.compile(r"\$(Date|LastChangedDate)[^\$]+\$")
+
+
+def main():
+  parser = ArgumentParser()
+  parser.add_argument(
+      "-v", "--verbose", action="store_true", help="enable debug logging")
+  parser.add_argument(
+      "-c",
+      "--check",
+      metavar="reference_file",
+      help="read checksums from reference_file and " +
+      "check they match checksums of llvm_path.")
+  parser.add_argument(
+      "--partial",
+      action="store_true",
+      help="ignore projects from reference_file " +
+      "that are not checked out in llvm_path.")
+  parser.add_argument(
+      "--multi_dir",
+      action="store_true",
+      help="indicates llvm_path contains llvm, checked out " +
+      "into multiple directories, as opposed to a " +
+      "typical single source tree checkout.")
+  parser.add_argument("llvm_path")
+
+  args = parser.parse_args()
+  if args.check is not None:
+    with open(args.check, "r") as f:
+      reference_checksums = ReadLLVMChecksums(f)
+  else:
+    reference_checksums = None
+
+  if args.verbose:
+    logging.basicConfig(level=logging.DEBUG)
+
+  llvm_projects = CreateLLVMProjects(not args.multi_dir)
+  checksums = ComputeLLVMChecksums(args.llvm_path, llvm_projects)
+
+  if reference_checksums is None:
+    WriteLLVMChecksums(checksums, sys.stdout)
+    sys.exit(0)
+
+  if not ValidateChecksums(reference_checksums, checksums, args.partial):
+    sys.stdout.write("Checksums differ.\nNew checksums:\n")
+    WriteLLVMChecksums(checksums, sys.stdout)
+    sys.stdout.write("Reference checksums:\n")
+    WriteLLVMChecksums(reference_checksums, sys.stdout)
+    sys.exit(1)
+  else:
+    sys.stdout.write("Checksums match.")
+
+
+def ComputeLLVMChecksums(root_path, projects):
+  """Compute checksums for LLVM sources checked out using svn.
+
+  Args:
+    root_path: a directory of llvm checkout.
+    projects: a list of LLVMProject instances, which describe checkout paths,
+      relative to root_path.
+
+  Returns:
+    A dict mapping from project name to project checksum.
+  """
+  hash_algo = hashlib.sha256
+
+  def collapse_svn_substitutions(contents):
+    # Replace svn substitutions for $Date$ and $LastChangedDate$.
+    # Unfortunately, these are locale-specific.
+    return SVN_DATES_REGEX.sub("$\1$", contents)
+
+  def read_and_collapse_svn_subsitutions(file_path):
+    with open(file_path, "rb") as f:
+      contents = f.read()
+      new_contents = collapse_svn_substitutions(contents)
+      if contents != new_contents:
+        logging.debug("Replaced svn keyword substitutions in %s", file_path)
+        logging.debug("\n\tBefore\n%s\n\tAfter\n%s", contents, new_contents)
+      return new_contents
+
+  project_checksums = dict()
+  # Hash each project.
+  for proj in projects:
+    project_root = os.path.join(root_path, proj.relpath)
+    if not os.path.exists(project_root):
+      logging.info("Folder %s doesn't exist, skipping project %s", proj.relpath,
+                   proj.name)
+      continue
+
+    files = list()
+
+    def add_file_hash(file_path):
+      if os.path.islink(file_path) and not os.path.exists(file_path):
+        content = os.readlink(file_path)
+      else:
+        content = read_and_collapse_svn_subsitutions(file_path)
+      hasher = hash_algo()
+      hasher.update(content)
+      file_digest = hasher.hexdigest()
+      logging.debug("Checksum %s for file %s", file_digest, file_path)
+      files.append((file_path, file_digest))
+
+    logging.info("Computing checksum for %s", proj.name)
+    WalkProjectFiles(root_path, projects, proj, add_file_hash)
+
+    # Compute final checksum.
+    files.sort(key=lambda x: x[0])
+    hasher = hash_algo()
+    for file_path, file_digest in files:
+      file_path = os.path.relpath(file_path, project_root)
+      hasher.update(file_path)
+      hasher.update(file_digest)
+    project_checksums[proj.name] = hasher.hexdigest()
+  return project_checksums
+
+
+def WriteLLVMChecksums(checksums, f):
+  """Writes checksums to a text file.
+
+  Args:
+    checksums: a dict mapping from project name to project checksum (result of
+      ComputeLLVMChecksums).
+    f: a file object to write into.
+  """
+
+  for proj in sorted(checksums.keys()):
+    f.write("{} {}\n".format(checksums[proj], proj))
+
+
+def ReadLLVMChecksums(f):
+  """Reads checksums from a text file, produced by WriteLLVMChecksums.
+
+  Returns:
+    A dict, mapping from project name to project checksum.
+  """
+  checksums = {}
+  while True:
+    line = f.readline()
+    if line == "":
+      break
+    checksum, proj = line.split()
+    checksums[proj] = checksum
+  return checksums
+
+
+def ValidateChecksums(reference_checksums,
+                      new_checksums,
+                      allow_missing_projects=False):
+  """Validates that reference_checksums and new_checksums match.
+
+  Args:
+    reference_checksums: a dict of reference checksums, mapping from a project
+      name to a project checksum.
+    new_checksums: a dict of checksums to be checked, mapping from a project
+      name to a project checksum.
+    allow_missing_projects:
+      When True, reference_checksums may contain more projects than
+        new_checksums. Projects missing from new_checksums are ignored.
+      When False, new_checksums and reference_checksums must contain checksums
+        for the same set of projects. If there is a project in
+        reference_checksums, missing from new_checksums, ValidateChecksums
+        will return False.
+
+  Returns:
+    True, if checksums match with regards to allow_missing_projects flag value.
+    False, otherwise.
+  """
+  if not allow_missing_projects:
+    if len(new_checksums) != len(reference_checksums):
+      return False
+
+  for proj, checksum in new_checksums.iteritems():
+    # We never computed a checksum for this project.
+    if proj not in reference_checksums:
+      return False
+    # Checksum did not match.
+    if reference_checksums[proj] != checksum:
+      return False
+
+  return True
+
+
+if __name__ == "__main__":
+  main()

utils/docker/scripts/llvm_checksum/project_tree.py

@@ -0,0 +1,95 @@
+"""Contains helper functions to compute checksums for LLVM checkouts.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import logging
+import os
+import os.path
+import sys
+
+
+class LLVMProject(object):
+  """An LLVM project with a descriptive name and a relative checkout path.
+  """
+
+  def __init__(self, name, relpath):
+    self.name = name
+    self.relpath = relpath
+
+  def is_subproject(self, other_project):
+    """ Check if self is checked out as a subdirectory of other_project.
+    """
+    return self.relpath.startswith(other_project.relpath)
+
+
+def WalkProjectFiles(checkout_root, all_projects, project, visitor):
+  """ Walk over all files inside a project without recursing into subprojects, '.git' and '.svn' subfolders.
+
+    checkout_root: root of the LLVM checkout.
+    all_projects: projects in the LLVM checkout.
+    project: a project to walk the files of. Must be inside all_projects.
+    visitor: a function called on each visited file.
+  """
+  assert project in all_projects
+
+  ignored_paths = set()
+  for other_project in all_projects:
+    if other_project != project and other_project.is_subproject(project):
+      ignored_paths.add(os.path.join(checkout_root, other_project.relpath))
+
+  def raise_error(err):
+    raise err
+
+  project_root = os.path.join(checkout_root, project.relpath)
+  for root, dirs, files in os.walk(project_root, onerror=raise_error):
+    dirs[:] = [
+        d for d in dirs
+        if d != ".svn" and d != ".git" and
+        os.path.join(root, d) not in ignored_paths
+    ]
+    for f in files:
+      visitor(os.path.join(root, f))
+
+
+def CreateLLVMProjects(single_tree_checkout):
+  """Returns a list of LLVMProject instances, describing relative paths of a typical LLVM checkout.
+
+  Args:
+    single_tree_checkout:
+      When True, relative paths for each project points to a typical single
+        source tree checkout.
+      When False, relative paths for each projects points to a separate
+        directory. However, clang-tools-extra is an exception, its relative path
+        will always be 'clang/tools/extra'.
+  """
+  # FIXME: cover all of llvm projects.
+
+  # Projects that reside inside 'projects/' in a single source tree checkout.
+  ORDINARY_PROJECTS = [
+      "compiler-rt", "dragonegg", "libcxx", "libcxxabi", "libunwind",
+      "parallel-libs", "test-suite"
+  ]
+  # Projects that reside inside 'tools/' in a single source tree checkout.
+  TOOLS_PROJECTS = ["clang", "lld", "lldb", "llgo"]
+
+  if single_tree_checkout:
+    projects = [LLVMProject("llvm", "")]
+    projects += [
+        LLVMProject(p, os.path.join("projects", p)) for p in ORDINARY_PROJECTS
+    ]
+    projects += [
+        LLVMProject(p, os.path.join("tools", p)) for p in TOOLS_PROJECTS
+    ]
+    projects.append(
+        LLVMProject("clang-tools-extra",
+                    os.path.join("tools", "clang", "tools", "extra")))
+  else:
+    projects = [LLVMProject("llvm", "llvm")]
+    projects += [LLVMProject(p, p) for p in ORDINARY_PROJECTS]
+    projects += [LLVMProject(p, p) for p in TOOLS_PROJECTS]
+    projects.append(
+        LLVMProject("clang-tools-extra", os.path.join("clang", "tools",
+                                                      "extra")))
+  return projects