[libcxx-commits] [libcxx] 92e563b - [libc++] SSH: Create a tarball of dependencies and scp that instead

[Louis Dionne via libcxx-commits]

Author: Louis Dionne
Date: 2020-04-01T16:38:21-04:00
New Revision: 92e563bc05d425c4667c44ce363f5db39cc3a25b

URL: https://github.com/llvm/llvm-project/commit/92e563bc05d425c4667c44ce363f5db39cc3a25b
DIFF: https://github.com/llvm/llvm-project/commit/92e563bc05d425c4667c44ce363f5db39cc3a25b.diff

LOG: [libc++] SSH: Create a tarball of dependencies and scp that instead

The benefit of doing this is that we can now handle directories that
contain symlinks and other arbitrary things, such as the static_test_env
required by filesystem tests.

As a fly-by fix, we also accumulate several commands to perform over SSH
and execute them at once instead of SSHing several times. This should be
faster on average.

Added: 
    

Modified: 
    libcxx/utils/ssh.py

Removed: 
    


################################################################################
diff  --git a/libcxx/utils/ssh.py b/libcxx/utils/ssh.py
index 82fc88441edb..f6b9574e39b2 100644
--- a/libcxx/utils/ssh.py
+++ b/libcxx/utils/ssh.py
@@ -18,6 +18,8 @@
 import posixpath
 import subprocess
 import sys
+import tarfile
+import tempfile
 
 
 def main():
@@ -35,7 +37,7 @@ def main():
     commandLine = remaining[1:] # Skip the '--'
 
     ssh = lambda command: ['ssh', '-oBatchMode=yes', args.host, command]
-    scp = lambda src, dst: ['scp', '-oBatchMode=yes', '-p', '-r', src, '{}:{}'.format(args.host, dst)]
+    scp = lambda src, dst: ['scp', '-oBatchMode=yes', src, '{}:{}'.format(args.host, dst)]
 
     # Create a temporary directory where the test will be run.
     tmp = subprocess.check_output(ssh('mktemp -d /tmp/libcxx.XXXXXXXXXX'), universal_newlines=True).strip()
@@ -48,54 +50,58 @@ def main():
     # for there to be no such executable, for example in the case of a .sh.cpp
     # test.
     isTestExe = lambda exe: exe.endswith('.tmp.exe') and os.path.exists(exe)
-    testExeOnRemote = lambda exe: posixpath.join(tmp, os.path.basename(exe))
+    pathOnRemote = lambda file: posixpath.join(tmp, os.path.basename(file))
 
     try:
         # Do any necessary codesigning of test-executables found in the command line.
         if args.codesign_identity:
             for exe in filter(isTestExe, commandLine):
-                rc = subprocess.call(['xcrun', 'codesign', '-f', '-s', args.codesign_identity, exe], env={})
-                if rc != 0:
-                    sys.stderr.write('Failed to codesign: {}'.format(exe))
-                    return rc
-
-        # Ensure the test dependencies exist and scp them to the temporary directory.
-        # Test dependencies can be either files or directories, so the `scp` command
-        # needs to use `-r`.
-        for dep in args.dependencies:
-            if not os.path.exists(dep):
-                sys.stderr.write('Missing file or directory {} marked as a dependency of a test'.format(dep))
-                return 1
-            rc = subprocess.call(scp(dep, tmp))
-            if rc != 0:
-                sys.stderr.write('Failed to copy dependency "{}" to remote host'.format(dep))
-                return rc
+                subprocess.check_call(['xcrun', 'codesign', '-f', '-s', args.codesign_identity, exe], env={})
+
+        # Ensure the test dependencies exist, tar them up and copy the tarball
+        # over to the remote host.
+        with tempfile.NamedTemporaryFile(suffix='.tar') as tmpTar:
+            with tarfile.open(fileobj=tmpTar, mode='w') as tarball:
+                for dep in args.dependencies:
+                    if not os.path.exists(dep):
+                        sys.stderr.write('Missing file or directory "{}" marked as a dependency of a test'.format(dep))
+                        return 1
+                    tarball.add(dep, arcname=os.path.basename(dep))
+
+            remoteTarball = pathOnRemote(tmpTar.name)
+            tmpTar.flush()
+            subprocess.check_call(scp(tmpTar.name, remoteTarball))
+
+        # Untar the dependencies in the temporary directory and remove the tarball.
+        remoteCommands = [
+            'tar -xf {} -C {}'.format(remoteTarball, tmp),
+            'rm {}'.format(remoteTarball)
+        ]
 
         # Make sure all test-executables in the remote command line have 'execute'
         # permissions on the remote host. The host that compiled the test-executable
         # might not have a notion of 'executable' permissions.
-        for exe in map(testExeOnRemote, filter(isTestExe, commandLine)):
-            rc = subprocess.call(ssh('chmod +x {}'.format(exe)))
-            if rc != 0:
-                sys.stderr.write('Failed to chmod +x test-executable "{}" on the remote host'.format(exe))
-                return rc
+        for exe in map(pathOnRemote, filter(isTestExe, commandLine)):
+            remoteCommands.append('chmod +x {}'.format(exe))
 
         # Execute the command through SSH in the temporary directory, with the
         # correct environment. We tweak the command line to run it on the remote
         # host by transforming the path of test-executables to their path in the
         # temporary directory, where we know they have been copied when we handled
         # test dependencies above.
-        commands = [
+        remoteCommands += [
             'cd {}'.format(tmp),
             'export {}'.format(' '.join(args.env)),
-            ' '.join(testExeOnRemote(x) if isTestExe(x) else x for x in commandLine)
+            ' '.join(pathOnRemote(x) if isTestExe(x) else x for x in commandLine)
         ]
-        rc = subprocess.call(ssh(' && '.join(commands)))
+
+        # Finally, SSH to the remote host and execute all the commands.
+        rc = subprocess.call(ssh(' && '.join(remoteCommands)))
         return rc
 
     finally:
         # Make sure the temporary directory is removed when we're done.
-        subprocess.call(ssh('rm -r {}'.format(tmp)))
+        subprocess.check_call(ssh('rm -r {}'.format(tmp)))
 
 
 if __name__ == '__main__':