adds system for regression tests

 * adds mitmproxy (https://github.com/saironiq/mitmproxy), using it in testing
 * tests for fence_apc, fence_docker, fence_ipmilan

Author: ondrejmular

fence/agents/cisco_ucs/fence_cisco_ucs.py

@@ -7,6 +7,7 @@
 sys.path.append("@FENCEAGENTSLIBDIR@")
 from fencing import *
 from fencing import fail, EC_STATUS, EC_LOGIN_DENIED, run_delay
+import fencing_pycurl
 
 #BEGIN_VERSION_GENERATION
 RELEASE_VERSION="New Cisco UCS Agent - test release on steroids"
@@ -85,7 +86,7 @@ def send_command(opt, command, timeout):
 	url += "//" + opt["--ip"] + ":" + str(opt["--ipport"]) + "/nuova"
 
 	## send command through pycurl
-	conn = pycurl.Curl()
+	conn = fencing_pycurl.FencingPyCurl()
 	web_buffer = StringIO.StringIO()
 	conn.setopt(pycurl.URL, url)
 	conn.setopt(pycurl.HTTPHEADER, ["Content-type: text/xml"])

fence/agents/docker/fence_docker.py

@@ -9,6 +9,7 @@
 
 sys.path.append("@FENCEAGENTSLIBDIR@")
 from fencing import fail_usage, all_opt, fence_action, atexit_handler, check_input, process_input, show_docs, run_delay
+import fencing_pycurl
 
 #BEGIN_VERSION_GENERATION
 RELEASE_VERSION = ""
@@ -50,7 +51,7 @@ def get_list(conn, options):
 
 def send_cmd(options, cmd, post = False):
 	url = "http%s://%s:%s/v1.11/%s" % ("s" if "--ssl" in options else "", options["--ip"], options["--ipport"], cmd)
-	conn = pycurl.Curl()
+	conn = fencing_pycurl.FencingPyCurl()
 	output_buffer = StringIO.StringIO()
 	if logging.getLogger().getEffectiveLevel() < logging.WARNING:
 		conn.setopt(pycurl.VERBOSE, True)

fence/agents/lib/Makefile.am

@@ -1,12 +1,12 @@
 MAINTAINERCLEANFILES	= Makefile.in
 
-TARGET			= fencing.py fencing_snmp.py
+TARGET			= fencing.py fencing_snmp.py fencing_pycurl.py
 
 if BUILD_XENAPILIB
 TARGET			+= XenAPI.py
 endif
 
-SRC			= fencing.py.py fencing_snmp.py.py XenAPI.py.py check_used_options.py
+SRC			= fencing.py.py fencing_snmp.py.py XenAPI.py.py check_used_options.py fencing_pycurl.py.py
 
 XSL			= fence2man.xsl fence2rng.xsl fence2wiki.xsl
 

fence/agents/lib/fencing.py.py

@@ -872,6 +872,7 @@ def fence_login(options, re_login_string=r"(login\s*: )|((?!Last )Login Name:  )
 	return conn
 
 def is_executable(path):
+	path = shlex.split(path)[0]
 	if os.path.exists(path):
 		stats = os.stat(path)
 		if stat.S_ISREG(stats.st_mode) and os.access(path, os.X_OK):

fence/agents/lib/fencing_pycurl.py.py

@@ -0,0 +1,156 @@
+__author__ = 'Ondrej Mular <[email protected]>'
+__all__ = ["FencingPyCurl"]
+
+import pycurl
+import sys
+import atexit
+import json
+import StringIO
+import time
+import logging
+import pprint
+
+## do not add code here.
+#BEGIN_VERSION_GENERATION
+RELEASE_VERSION = ""
+REDHAT_COPYRIGHT = ""
+BUILD_DATE = ""
+#END_VERSION_GENERATION
+
+class FencingPyCurl():
+	active = False
+	input_file = None
+	output_file = None
+	options = None
+	actions = None
+	request_index = 0
+	output_buffer = None
+	write_function = None
+	pycurl_obj = None
+
+	def __init__(self):
+		if not FencingPyCurl.active and (FencingPyCurl.input_file or FencingPyCurl.output_file):
+			FencingPyCurl.active = True
+			logging.debug("FencingPyCurl is active now")
+		if FencingPyCurl.active:
+			self.options = {
+				"request": {},
+				"response": {},
+				"time": {}
+			}
+
+			self.output_buffer = StringIO.StringIO()
+			if FencingPyCurl.actions is None:
+				if FencingPyCurl.input_file:
+					logging.debug("Reading input file.")
+
+					try:
+						with open(FencingPyCurl.input_file, "r") as f:
+							FencingPyCurl.actions = json.load(f)
+					except Exception as e:
+						logging.debug("Reading input file (%s) failed: %s" % (FencingPyCurl.input_file, e.message))
+
+				if FencingPyCurl.output_file:
+					logging.debug("output file detected")
+					if not FencingPyCurl.actions:
+						FencingPyCurl.actions = []
+						FencingPyCurl.request_index = 0
+
+		self.pycurl_obj = pycurl.Curl()
+
+	def setopt(self, opt, value):
+		if FencingPyCurl.active:
+			if opt == pycurl.WRITEFUNCTION:
+				self.write_function = value
+				value = self.output_buffer.write
+			else:
+				self.options["request"][str(opt)] = value
+		return self.pycurl_obj.setopt(opt, value)
+
+	def perform(self):
+		if FencingPyCurl.active:
+			if FencingPyCurl.input_file:
+				perform_start = time.time()
+				if self.options["request"] == FencingPyCurl.actions[FencingPyCurl.request_index]["request"]:
+					self.options["response"] = FencingPyCurl.actions[FencingPyCurl.request_index]["response"]
+					if self.write_function:
+						self.write_function(self.options["response"]["output"])
+					diff = FencingPyCurl.actions[FencingPyCurl.request_index]["time"]["perform_duration"] - (time.time() - perform_start)
+					if diff > 0:
+						logging.debug("sleeping for: %s" % str(diff))
+						time.sleep(diff)
+					FencingPyCurl.last_request_time = time.time()
+				else:
+					print "Request:"
+					pprint.pprint(self.options["request"])
+					print "Expected:"
+					pprint.pprint(FencingPyCurl.actions[FencingPyCurl.request_index]["request"])
+					raise Exception("Invalid request")
+			else:
+				start_time = time.time()
+				self.pycurl_obj.perform()
+				self.options["time"]["perform_duration"] = FencingPyCurl.last_request_time - start_time
+				self.options["response"]["output"] = self.output_buffer.getvalue()
+				if self.write_function:
+					self.write_function(self.options["response"]["output"])
+				if FencingPyCurl.output_file:
+					FencingPyCurl.actions.append(self.options)
+			FencingPyCurl.request_index += 1
+		else:
+			return self.pycurl_obj.perform()
+
+	def unsetopt(self, opt):
+		if FencingPyCurl.active:
+			del self.options["request"][str(opt)]
+		return self.pycurl_obj.unsetopt(opt)
+
+	def getinfo(self, opt):
+		value = self.pycurl_obj.getinfo(opt)
+		if FencingPyCurl.active:
+			if FencingPyCurl.input_file:
+				value = self.options["response"][str(opt)]
+			else:
+				self.options["response"][opt] = value
+		return value
+
+	def reset(self):
+		if FencingPyCurl.active:
+			self.options.clear()
+		return self.pycurl_obj.reset()
+
+	def close(self):
+		return self.pycurl_obj.close()
+
+	@staticmethod
+	def save_log_to_file():
+		if FencingPyCurl.output_file and FencingPyCurl.actions:
+			logging.debug("Writing log to file: %s" % FencingPyCurl.output_file)
+			try:
+				with open(FencingPyCurl.output_file, "w") as f:
+					json.dump(FencingPyCurl.actions, f, sort_keys=True, indent=4, separators=(',', ': '))
+			except Exception as e:
+				logging.debug("Writing log to file (%s) failed: %s" % (FencingPyCurl.output_file, e.message))
+
+
+def get_and_remove_arg(arg, has_value=True):
+	logging.debug("Getting arg: %s (has_value: %s)" % (arg, str(has_value)))
+	if not has_value:
+		if arg in sys.argv:
+			sys.argv.remove(arg)
+			logging.debug("%s: True" % arg)
+			return True
+	if arg in sys.argv:
+		index = sys.argv.index(arg)
+		sys.argv.remove(arg)
+		if len(sys.argv) > index:
+			value = sys.argv[index]
+			sys.argv.remove(value)
+			logging.debug("%s: %s" % (arg, value))
+			return value
+	return None
+
+FencingPyCurl.input_file = get_and_remove_arg("--fencing_pycurl-log-in")
+FencingPyCurl.output_file = get_and_remove_arg("--fencing_pycurl-log-out")
+
+if FencingPyCurl.output_file:
+	atexit.register(FencingPyCurl.save_log_to_file)

fence/agents/pve/fence_pve.py

@@ -13,6 +13,7 @@
 import logging
 sys.path.append("@FENCEAGENTSLIBDIR@")
 from fencing import fail, EC_LOGIN_DENIED, atexit_handler, all_opt, check_input, process_input, show_docs, fence_action, run_delay
+import fencing_pycurl
 
 #BEGIN_VERSION_GENERATION
 RELEASE_VERSION=""
@@ -93,7 +94,7 @@ def get_ticket(options):
 
 def send_cmd(options, cmd, post=None):
 	url = options["url"] + cmd
-	conn = pycurl.Curl()
+	conn = fencing_pycurl.FencingPyCurl()
 	output_buffer = StringIO.StringIO()
 	if logging.getLogger().getEffectiveLevel() < logging.WARNING:
 		conn.setopt(pycurl.VERBOSE, True)

mitm/mitmproxy-0.1/INTERNAL.md

@@ -0,0 +1,140 @@
+MITMPROXY LIBRARY INTERNALS
+===========================
+
+
+Introduction
+------------
+The `mitmproxy` library consists of 2 modules: `mitmproxy.py` and `sshdebug.py`.
+Module `mitmproxy.py` contains implementation of proxy and replay servers
+for supported network protocols. Module `sshdebug.py` serves for more human-friendly
+debugging output of unencrypted SSH messages.
+
+The library is built on the Python Twisted network framework - see the project homepage
+at [twistedmatrix.com](http://twistedmatrix.com/).
+
+
+Supported network protocols
+---------------------------
+* Telnet
+* HTTP
+* HTTPS/SSL
+* SSH
+* SNMP
+
+
+MITM proxy server in general
+----------------------------
+```
+                +----------------------MITM PROXY---------------------+
+                | +--------------+     +-------+     +--------------+ |
+                | |    (receive) | <<< | Queue | <<< | (transmit)   | |
++--------+      | |              |     +-------+     |              | |      +--------+
+| Client | <--> | | Proxy Server |                   | Proxy Client | | <--> | Server |
++--------+      | |              |     +-------+     |              | |      +--------+
+                | |   (transmit) | >>> | Queue | >>> | (receive)    | |
+                | +--------------+     +-------+     +--------------+ |
+                +-----------------------------------------------------+
+```
+
+As you can see in the above diagram, the MITM proxy has 2 componets - proxy server
+and proxy client. These components communicate internally via deferred queues
+and all of the communication is logged for later use by the replay server.
+For details about deferred queues RTFM at [defer-intro](http://twistedmatrix.com/documents/current/core/howto/defer-intro.html).
+
+More info about TCP/UDP clients/servers in Twisted is available on the Twisted core [doc pages](http://twistedmatrix.com/documents/current/core/howto/index.html).
+Most of the proxies are quite similar - one exception is SSH because it has multiple layers.
+MITM SSH proxy and replay servers are using implementation of SSHv2 protocol
+from `twisted.conch.ssh` package. There are some examples of SSH clients and
+servers on twisted conch documentation pages:
+[conch doc](http://twistedmatrix.com/documents/current/conch/index.html).
+Another `twisted.conch.ssh` howto can be found here:
+[ticket-5474](http://twistedmatrix.com/trac/ticket/5474)
+and
+[ticket-6001](http://twistedmatrix.com/trac/ticket/6001).
+
+
+Notes on SSH
+------------
+### SSH keypairs
+SSH proxy/replay requires SSH keypairs even when they are not being used,
+so you should generate them with either `mitmkeygen` or by hand (and specify
+their location on command-line if you put them in non-default location).
+
+### Communication
+The communication between proxy components starts during authentication.
+Proxy server first negotiates authentication method with the client
+and then forwards it to the proxy client, which in turn tries authenticating
+against the real server. Then the proxy client informs proxy server about
+the authentication result. After SSH transport layer connection is established,
+the proxy simply forwards (decrypts and re-encrypts) connection layer packets.
+The forwarding is done by the `mitmproxy.ProxySSHConnection` class which is a
+subclass of `ssh.connection.SSHConnection`. The `mitmproxy.ProxySSHConnection` class simply
+overrides the `packetReceived` method which puts received packets into a queue and logs
+the SSH channel data (`SSH_MSG_CHANNEL_DATA` message type), which is assumed to be
+the interactive shell channel.
+
+### Authentication
+The client connects to proxy, the proxy in turn creates a connection
+to real server and forwards the SSH banner to the client. The client then chooses
+an authentication method to use and authenticates against the _proxy_.
+In case of password auth the proxy simply forwards it to the real server.
+With public key auth, however, the proxy needs to have the corresponding private
+key to be able to authenticate client - so the proxy has its own keypair
+to use for client connection. This also means that the proxy has to have the client's
+keypair (or any other keypair that is accepted as valid for the given user on the real server).
+Thus for the proxy to work with pubkey auth you need to add the public key of _proxy_ to
+the list of allowed keys for the give user at the given real server (usually ~/.ssh/authorized_keys).
+
+
+The proxy server uses Twisted's [Pluggable Authentication](http://twistedmatrix.com/documents/current/core/howto/cred.html) system.
+Proxy authentication is implemented in these classes:
+* ProxySSHUserAuthServer
+* ProxySSHUserAuthClient
+* SSHCredentialsChecker
+
+
+Proxy server side is implemented mostly in the `SSHCredentialsChecker` class.
+The `ProxySSHUserAuthServer` is a hack to be able to properly end the communication.
+The authentication result is evaluted in callback method `SSHCredentialsChecker.is_auth_succes`.
+There are three possible results:
+* auth succeeded
+* auth failed, more auth methods available - try another one
+* auth failed, no more auth methods - disconnect
+
+
+Proxy client auth is implemented in `ProxySSHUserAuthClient`.
+It sends/receives information to/from proxy server through deferred queues.
+After successful auth the ssh-connection service is started.
+
+
+There are some issues with proxy auth:
+* proxy and real client auth method attempt order must be the same
+* real server might support less auth methods than proxy
+
+First issue is solved by sending the name of current auth method used by client to proxy.
+Second issue is solved by pretending method failure and waiting for another auth method.
+
+The proxy tries to be as transparent as possible - everything depends only on server and client configuration
+(eg. the number of allowed password auth attemps) - well, at least it *should*. ;)
+
+
+### SSH Replay server
+SSH replay server always successfully authenticates client on the first authentication method attempt.
+The replay server is implemented in `mitmproxy.SSHReplayServerProtocol` and it is connected to the SSH service in
+`mitmproxy.ReplayAvatar` class.
+
+### SSH proxy drawbacks
+The proxy only supports logging of SSH sessions with _only one_ open channel.
+It does not support other SSH features like port forwarding.
+
+
+Notes about SNMP proxy and replay server
+----------------------------------------
+SNMP proxy is ordinary UDP server that intercepts and forwards UDP packets.
+Because of UDP protocol we do not know when the communication ends.
+You can save the PID of SNMP proxy process and when the client ends you can terminate the proxy.
+SNMP replay reads communication from log and compares received packets with expected packets.
+It copes with different request IDs in packets.
+There are functions for extracting and replacing the request-id from/in packets - `snmp_extract_request_id` and `snmp_replace_request_id`.
+SNMP replay server works only with UDP.
+