[Lldb-commits] [lldb] r254023 - swig_bot remote path connection / preliminary implementation.
Zachary Turner via lldb-commits
lldb-commits at lists.llvm.org
Tue Nov 24 13:35:32 PST 2015
Author: zturner
Date: Tue Nov 24 15:35:32 2015
New Revision: 254023
URL: http://llvm.org/viewvc/llvm-project?rev=254023&view=rev
Log:
swig_bot remote path connection / preliminary implementation.
With this patch, the client will package up all the required
inputs into a compressed zip file, establish a connection to the
server, send the input to the server, and wait for the server to
send a response (in this case the response is just echoed back to
the client).
This gets the network communication in place, and in a subsequent
patch I will follow up with the code that actually runs swig on
the server and sends back the output instead of echoing back the
input.
Added:
lldb/trunk/packages/Python/lldbsuite/support/sockutil.py
Modified:
lldb/trunk/scripts/swig_bot.py
lldb/trunk/scripts/swig_bot_lib/client.py
lldb/trunk/scripts/swig_bot_lib/local.py
lldb/trunk/scripts/swig_bot_lib/server.py
Added: lldb/trunk/packages/Python/lldbsuite/support/sockutil.py
URL: http://llvm.org/viewvc/llvm-project/lldb/trunk/packages/Python/lldbsuite/support/sockutil.py?rev=254023&view=auto
==============================================================================
--- lldb/trunk/packages/Python/lldbsuite/support/sockutil.py (added)
+++ lldb/trunk/packages/Python/lldbsuite/support/sockutil.py Tue Nov 24 15:35:32 2015
@@ -0,0 +1,23 @@
+"""
+ The LLVM Compiler Infrastructure
+
+This file is distributed under the University of Illinois Open Source
+License. See LICENSE.TXT for details.
+
+Helper functions for working with sockets.
+"""
+
+# Python modules:
+import io
+import socket
+
+# LLDB modules
+import use_lldb_suite
+
+def recvall(sock, size):
+ bytes = io.BytesIO()
+ while size > 0:
+ this_result = sock.recv(size)
+ bytes.write(this_result)
+ size -= len(this_result)
+ return bytes.getvalue()
Modified: lldb/trunk/scripts/swig_bot.py
URL: http://llvm.org/viewvc/llvm-project/lldb/trunk/scripts/swig_bot.py?rev=254023&r1=254022&r2=254023&view=diff
==============================================================================
--- lldb/trunk/scripts/swig_bot.py (original)
+++ lldb/trunk/scripts/swig_bot.py Tue Nov 24 15:35:32 2015
@@ -1,11 +1,63 @@
#!/usr/bin/env python
+"""
+SWIG generation top-level script. Supports both local and remote generation
+of SWIG bindings for multiple languages.
+"""
+
# Python modules
+import argparse
+import logging
import sys
+import traceback
# LLDB modules
import use_lldb_suite
+def process_args(args):
+ parser = argparse.ArgumentParser(
+ description='Run swig-bot client or server.')
+
+ # Arguments to control whether swig-bot runs as a client or server.
+ parser.add_argument(
+ "--mode",
+ required=True,
+ choices=["client", "server"],
+ help="Run swig_bot in either client or server mode.")
+
+ # Arguments to control logging verbosity.
+ parser.add_argument(
+ "--verbose", "-v",
+ action="store_true",
+ default=False,
+ help="Increase logging verbosity level.")
+
+ (options, remaining) = parser.parse_known_args(args)
+ # Set logging level.
+ if options.verbose:
+ log_level = logging.DEBUG
+ else:
+ log_level = logging.NOTSET
+ logging.basicConfig(level=log_level)
+ logging.info("logging is using level: %d", log_level)
+
+ return (options, remaining)
+
if __name__ == "__main__":
- from swig_bot_lib import client
- client.run(sys.argv[1:])
+ (options, remaining) = process_args(sys.argv[1:])
+ try:
+ if options.mode == "client":
+ logging.info("Running swig_bot in client mode")
+ from swig_bot_lib import client
+ client.run(remaining)
+ elif options.mode == "server":
+ logging.info("Running swig_bot in server mode")
+ from swig_bot_lib import server
+ server.run(remaining)
+ else:
+ logging.error("Unknown mode specified. Expected client or server.")
+ sys.exit(-1)
+ except Exception as e:
+ error = traceback.format_exc()
+ logging.error("An error occurred running swig-bot.")
+ logging.error(error)
Modified: lldb/trunk/scripts/swig_bot_lib/client.py
URL: http://llvm.org/viewvc/llvm-project/lldb/trunk/scripts/swig_bot_lib/client.py?rev=254023&r1=254022&r2=254023&view=diff
==============================================================================
--- lldb/trunk/scripts/swig_bot_lib/client.py (original)
+++ lldb/trunk/scripts/swig_bot_lib/client.py Tue Nov 24 15:35:32 2015
@@ -1,5 +1,10 @@
#!/usr/bin/env python
+"""
+SWIG generation client. Supports both local and remote generation of SWIG
+bindings for multiple languages.
+"""
+
# Future imports
from __future__ import absolute_import
from __future__ import print_function
@@ -8,11 +13,20 @@ from __future__ import print_function
import argparse
import logging
import os
+import socket
+import struct
import sys
# LLDB modules
import use_lldb_suite
from lldbsuite.support import fs
+from lldbsuite.support import sockutil
+
+# package imports
+from . import local
+
+default_ip = "127.0.0.1"
+default_port = 8537
def process_args(args):
"""Returns options processed from the provided command line.
@@ -20,9 +34,14 @@ def process_args(args):
@param args the command line to process.
"""
+ # A custom action used by the --local command line option. It can be
+ # used with either 0 or 1 argument. If used with 0 arguments, it
+ # searches for a copy of swig located on the physical machine. If
+ # used with 1 argument, the argument is the path to a swig executable.
class FindLocalSwigAction(argparse.Action):
def __init__(self, option_strings, dest, **kwargs):
- super(FindLocalSwigAction, self).__init__(option_strings, dest, nargs='?', **kwargs)
+ super(FindLocalSwigAction, self).__init__(
+ option_strings, dest, nargs='?', **kwargs)
def __call__(self, parser, namespace, values, option_string=None):
swig_exe = None
if values is None:
@@ -31,17 +50,33 @@ def process_args(args):
swig_exe = values
setattr(namespace, self.dest, os.path.normpath(swig_exe))
+ # A custom action used by the --remote command line option. It can be
+ # used with either 0 or 1 arguments. If used with 0 arguments it chooses
+ # a default connection string. If used with one argument it is a string
+ # of the form `ip_address[:port]`. If the port is unspecified, the
+ # default port is used.
+ class RemoteIpAction(argparse.Action):
+ def __init__(self, option_strings, dest, **kwargs):
+ super(RemoteIpAction, self).__init__(
+ option_strings, dest, nargs='?', **kwargs)
+ def __call__(self, parser, namespace, values, option_string=None):
+ ip_port = None
+ if values is None:
+ ip_port = (default_ip, default_port)
+ else:
+ result = values.split(':')
+ if len(result)==1:
+ ip_port = (result[0], default_port)
+ elif len(result)==2:
+ ip_port = (result[0], int(result[1]))
+ else:
+ raise ValueError("Invalid connection string")
+ setattr(namespace, self.dest, ip_port)
+
# Setup the parser arguments that are accepted.
parser = argparse.ArgumentParser(
description='Generate SWIG bindings.')
- # Arguments to control logging verbosity.
- parser.add_argument(
- "--verbose", "-v",
- action="store_true",
- default=False,
- help="Increase logging verbosity level.")
-
parser.add_argument(
"--local",
action=FindLocalSwigAction,
@@ -52,7 +87,7 @@ def process_args(args):
parser.add_argument(
"--remote",
- action="store",
+ action=RemoteIpAction,
help=(
"Use the given connection string to connect to a remote "
"generation service"))
@@ -85,24 +120,42 @@ def process_args(args):
logging.error("Must specify either --local or --remote")
sys.exit(-3)
- # Set logging level based on verbosity count.
- if options.verbose:
- log_level = logging.DEBUG
- else:
- log_level = logging.NOTSET
- logging.basicConfig(level=log_level)
- logging.info("logging is using level: %d", log_level)
-
return options
+def establish_remote_connection(ip_port):
+ logging.debug("Creating socket...")
+ s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ logging.info("Connecting to server {} on port {}"
+ .format(ip_port[0], ip_port[1]))
+ s.connect(ip_port)
+ logging.info("Connection established...")
+ return s
+
+def transmit_data(connection, packed_input):
+ logging.info("Sending {} bytes of compressed data."
+ .format(len(packed_input)))
+ connection.sendall(struct.pack("!I", len(packed_input)))
+ connection.sendall(packed_input)
+ logging.info("Awaiting response.")
+ response_len = struct.unpack("!I", sockutil.recvall(connection, 4))[0]
+ logging.debug("Expecting {} byte response".format(response_len))
+ response = sockutil.recvall(connection, response_len)
+ return response
def run(args):
options = process_args(args)
if options.remote is None:
+ logging.info("swig bot client using local swig installation at '{}'"
+ .format(options.swig_executable))
if not os.path.isfile(options.swig_executable):
- logging.error("Swig executable '%s' does not exist." % options.swig_executable)
- from . import local
+ logging.error("Swig executable '{}' does not exist."
+ .format(options.swig_executable))
local.generate(options)
else:
- logging.error("Remote path is not yet implemented!")
+ logging.info("swig bot client using remote generation with server '{}'"
+ .format(options.remote))
+ packed_input = local.pack_input(options)
+ connection = establish_remote_connection(options.remote)
+ response = transmit_data(connection, packed_input)
+ logging.debug("Received {} byte response.".format(len(response)))
Modified: lldb/trunk/scripts/swig_bot_lib/local.py
URL: http://llvm.org/viewvc/llvm-project/lldb/trunk/scripts/swig_bot_lib/local.py?rev=254023&r1=254022&r2=254023&view=diff
==============================================================================
--- lldb/trunk/scripts/swig_bot_lib/local.py (original)
+++ lldb/trunk/scripts/swig_bot_lib/local.py Tue Nov 24 15:35:32 2015
@@ -1,5 +1,10 @@
#!/usr/bin/env python
+"""
+Shared functionality used by `client` and `server` when generating or preparing
+to generate SWIG on the local machine.
+"""
+
# Future imports
from __future__ import absolute_import
from __future__ import print_function
@@ -7,14 +12,57 @@ from __future__ import print_function
# Python modules
import argparse
import imp
+import io
import logging
import os
import subprocess
import sys
+import zipfile
# LLDB modules
import use_lldb_suite
+def pack_input(options):
+ logging.info("Creating input file package...")
+ zip_data = io.BytesIO()
+ zip_file = None
+ try:
+ # It's possible that a custom-built interpreter will not have the
+ # standard zlib module. If so, we can only store, not compress. By
+ # try to compress since we usually have a standard Python distribution.
+ zip_file = zipfile.ZipFile(zip_data, mode='w',
+ compression=zipfile.ZIP_DEFLATED)
+ except RuntimeError:
+ zip_file = zipfile.ZipFile(zip_data, mode='w',
+ compression=zipfile.ZIP_STORED)
+
+ filters = [("include/lldb", ".h"),
+ ("scripts", ".swig"),
+ ("scripts/Python", ".swig"),
+ ("scripts/interface", ".i")]
+ def filter_func(t):
+ subfolder = t[0]
+ ext = t[1]
+ full_path = os.path.normpath(os.path.join(options.src_root, subfolder))
+ candidates = [os.path.normpath(os.path.join(full_path, f))
+ for f in os.listdir(full_path)]
+ actual = filter(
+ lambda f : os.path.isfile(f) and os.path.splitext(f)[1] == ext,
+ candidates)
+ return (subfolder, map(lambda f : os.path.basename(f), actual))
+ archive_entries = map(filter_func, filters)
+
+ for entry in archive_entries:
+ subfolder = entry[0]
+ files = entry[1]
+ for file in files:
+ relative_path = os.path.normpath(os.path.join(subfolder, file))
+ full_path = os.path.normpath(
+ os.path.join(options.src_root, relative_path))
+ logging.info("{} -> {}".format(full_path, relative_path))
+ zip_file.write(full_path, relative_path)
+ return zip_data.getvalue()
+
def generate(options):
include_folder = os.path.join(options.src_root, "include")
in_file = os.path.join(options.src_root, "scripts", "lldb.swig")
@@ -43,7 +91,8 @@ def generate(options):
in_file
])
- logging.info("generating swig {} bindings into {}".format(lang, out_dir))
+ logging.info("generating swig {} bindings into {}"
+ .format(lang, out_dir))
logging.debug("swig command line: {}".format(swig_command))
try:
# Execute swig
@@ -54,5 +103,6 @@ def generate(options):
if swig_output is not None and len(swig_output) > 0:
logging.info("swig output: %s", swig_output)
except subprocess.CalledProcessError as e:
- logging.error("An error occurred executing swig. returncode={}".format(e.returncode))
+ logging.error("An error occurred executing swig. returncode={}"
+ .format(e.returncode))
logging.error(e.output)
Modified: lldb/trunk/scripts/swig_bot_lib/server.py
URL: http://llvm.org/viewvc/llvm-project/lldb/trunk/scripts/swig_bot_lib/server.py?rev=254023&r1=254022&r2=254023&view=diff
==============================================================================
--- lldb/trunk/scripts/swig_bot_lib/server.py (original)
+++ lldb/trunk/scripts/swig_bot_lib/server.py Tue Nov 24 15:35:32 2015
@@ -1,6 +1,80 @@
#!/usr/bin/env python
+"""
+SWIG generation server. Listens for connections from swig generation clients
+and runs swig in the requested fashion, sending back the results.
+"""
+
# Future imports
from __future__ import absolute_import
from __future__ import print_function
+# Python modules
+import argparse
+import logging
+import os
+import socket
+import struct
+import sys
+import traceback
+
+# LLDB modules
+import use_lldb_suite
+from lldbsuite.support import sockutil
+
+# package imports
+from . import local
+
+default_port = 8537
+
+def process_args(args):
+ # Setup the parser arguments that are accepted.
+ parser = argparse.ArgumentParser(description='SWIG generation server.')
+
+ parser.add_argument(
+ "--port",
+ action="store",
+ default=default_port,
+ help=("The local port to bind to"))
+
+ # Process args.
+ return parser.parse_args(args)
+
+def initialize_listening_socket(options):
+ logging.debug("Creating socket...")
+ s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+
+ logging.info("Binding to ip address '', port {}".format(options.port))
+ s.bind(('', options.port))
+
+ logging.debug("Putting socket in listen mode...")
+ s.listen()
+ return s
+
+def accept_once(sock, options):
+ logging.debug("Waiting for connection...")
+ client, addr = sock.accept()
+ logging.info("Received connection from {}".format(addr))
+ data_size = struct.unpack("!I", sockutil.recvall(client, 4))[0]
+ logging.debug("Expecting {} bytes of data from client".format(data_size))
+ data = sockutil.recvall(client, data_size)
+ logging.info("Received {} bytes of data from client".format(len(data)))
+
+ logging.info("Sending {} byte response".format(len(data)))
+ client.sendall(struct.pack("!I", len(data)))
+ client.sendall(data)
+
+def accept_loop(sock, options):
+ while True:
+ try:
+ accept_once(sock, options)
+ except Exception as e:
+ error = traceback.format_exc()
+ logging.error("An error occurred while processing the connection.")
+ logging.error(error)
+
+def run(args):
+ options = process_args(args)
+ sock = initialize_listening_socket(options)
+ accept_loop(sock, options)
+ return options
More information about the lldb-commits
mailing list