python-network
Python module for easy networking
Loading...
Searching...
No Matches
network.py
Go to the documentation of this file.
1# vim: set fileencoding=utf-8 foldmethod=marker :
2
3# {{{ Copyright 2013-2019 Bas Wijnen <wijnen@debian.org>
4# This program is free software: you can redistribute it and/or modify
5# it under the terms of the GNU Affero General Public License as
6# published by the Free Software Foundation, either version 3 of the
7# License, or(at your option) any later version.
8#
9# This program is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU Affero General Public License for more details.
13#
14# You should have received a copy of the GNU Affero General Public License
15# along with this program. If not, see <http://www.gnu.org/licenses/>.
16# }}}
17
18
23
24'''@file
25Python module for easy networking. This module intends to make networking
26easy. It supports tcp and unix domain sockets. Connection targets can be
27specified in several ways.
28'''
29
30'''@package network Python module for easy networking.
31This module intends to make networking easy. It supports tcp and unix domain
32sockets. Connection targets can be specified in several ways.
33'''
34
35# {{{ Imports.
36import math
37import sys
38import os
39import socket
40import select
41import re
42import time
43import inspect
44import fhs
45modulename = 'network'
46fhs.module_info(modulename, 'Networking made easy', '0.4', 'Bas Wijnen <wijnen@debian.org>')
47fhs.module_option(modulename, 'tls', 'default tls hostname for server sockets. The code may ignore this option. Set to - to request that tls is disabled on the server. If left empty, detects hostname.', default = '')
48import traceback
49
50try:
51 import ssl
52 have_ssl = True
53except:
54 have_ssl = False
55# }}}
56
57# {{{ Interface description
58# - connection setup
59# - connect to server
60# - listen on port
61# - when connected
62# - send data
63# - asynchronous read
64# - blocking read for data
65
66# implementation:
67# - Server: listener, creating Sockets on accept
68# - Socket: used for connection; symmetric
69# }}}
70
71if sys.version >= '3':
72 makestr = lambda x: str(x, 'utf8', 'replace') if isinstance(x, bytes) else x
73else:
74 makestr = lambda x: x
75
76log_output = sys.stderr
77log_date = False
78
79
86 global log_output, log_date
87 log_output = file
88 log_date = True
89# }}}
90
91
102def log(*message, filename = None, line = None, funcname = None, depth = 0):
103 t = time.strftime('%F %T' if log_date else '%T')
104 source = inspect.currentframe().f_back
105 for d in range(depth):
106 source = source.f_back
107 code = source.f_code
108 if filename is None:
109 filename = os.path.basename(code.co_filename)
110 if funcname is None:
111 funcname = code.co_name
112 if line is None:
113 line = source.f_lineno
114 for msg in message:
115 log_output.write(''.join(['%s %s:%s:%d:\t%s\n' % (t, filename, funcname, line, m) for m in str(msg).split('\n')]))
116 log_output.flush()
117# }}}
118
119
123def lookup(service):
124 if isinstance(service, int):
125 return service
126 try:
127 return socket.getservbyname(service)
128 except socket.error:
129 pass
130 return int(service)
131# }}}
132
133
140class _Fake:
141
145 def __init__(self, i, o = None):
146 self._i = i
147 self._o = o if o is not None else i
148
150 def close(self):
151 pass
152
154 def sendall(self, data):
155 while len(data) > 0:
156 fd = self._o if isinstance(self._o, int) else self._o.fileno()
157 ret = os.write(fd, data)
158 if ret >= 0:
159 data = data[ret:]
160 continue
161 log('network.py: Failed to write data')
162 traceback.print_exc()
163
165 def recv(self, maxsize):
166 #log('recv fake')
167 return os.read(self._i.fileno(), maxsize)
168
170 def fileno(self):
171 # For reading.
172 return self._i.fileno()
173# }}}
174
175
181def wrap(i, o = None):
182 return Socket(_Fake(i, o))
183# }}}
184
185
187class Socket:
188
204 def __init__(self, address, tls = None, disconnect_cb = None, remote = None, connections = None):
205
206 self.tls = tls
207
208 self.remote = remote
209
210 self.connections = connections
211 if self.connections is not None:
212 self.connections.add(self)
213
214 self.socket = None
215 self._disconnect_cb = disconnect_cb
216 self._event = None
217 self._linebuffer = b''
218 if isinstance(address, (_Fake, socket.socket)):
219 #log('new %d' % id(address))
220 self.socket = address
221 return
222 if isinstance(address, str):
223 r = re.match('^(?:([a-z0-9-]+)://)?([^:/?#]+)(?::([^:/?#]+))?([:/?#].*)?$', address)
224 if r is not None:
225 # Group 1: protocol or None
226 # Group 2: hostname
227 # Group 3: port
228 # Group 4: everything after the port (address, query string, etc)
229 protocol = r.group(1)
230 hostname = r.group(2)
231 port = r.group(3)
232 url = r.group(4)
233 #print('protocol', protocol, 'hostname', hostname, 'port', port, 'url', url)
234 if self.tls is None:
235 if protocol is None:
236 self.tls = False
237 else:
238 self.tls = protocol != 'ws' and protocol.endswith('s')
239 if port is None:
240 port = protocol
241 else:
242 port = None
243 if address.startswith('./') or address.startswith('/') or (port is None and '/' in address):
244 # Unix socket.
245 # TLS is ignored for those.
246 self.remote = address
247 self.socket = socket.socket(socket.AF_UNIX)
248 self.socket.connect(self.remote)
249 return
250 elif r is None:
251 # Probably an error, but attempt to parse address as port.
252 hostname = 'localhost'
253 port = address
254 else:
255 # Url (with possibly some missing parts)
256 if port is None:
257 hostname = 'localhost'
258 port = address
259 else:
260 hostname = 'localhost'
261 port = address
262 self.remote = (hostname, lookup(port))
263 #log('remote %s' % str(self.remote))
264 self._setup_connection()
265 # }}}
266
269 def _setup_connection(self):
270 self.socket = socket.create_connection(self.remote)
271 if self.tls is None:
272 try:
273 assert have_ssl
274 self.socket = ssl.wrap_socket(self.socket, ssl_version = ssl.PROTOCOL_TLSv1)
275 self.tls = True
276 except:
277 self.tls = False
278 self.socket = socket.create_connection(self.remote)
279 elif self.tls is True:
280 try:
281 assert have_ssl
282 self.socket = ssl.wrap_socket(self.socket, ssl_version = ssl.PROTOCOL_TLSv1)
283 except ssl.SSLError as e:
284 raise TypeError('Socket does not seem to support TLS: ' + str(e))
285 else:
286 self.tls = False
287 # }}}
288
291 def disconnect_cb(self, disconnect_cb):
292 self._disconnect_cb = disconnect_cb
293 # }}}
294
296 def close(self):
297 if not self.socket:
298 return b''
299 data = self.unread()
300 self.socket.close()
301 self.socket = None
302 if self.connections is not None:
303 self.connections.remove(self)
304 if self._disconnect_cb:
305 return self._disconnect_cb(self, data) or b''
306 return data
307 # }}}
308
313 def send(self, data):
314 if self.socket is None:
315 return
316 #print 'sending %s' % repr(data)
317 try:
318 self.socket.sendall(data)
319 except BrokenPipeError:
320 self.close()
321 # }}}
322
328 def sendline(self, data):
329 if self.socket is None:
330 return
331 #print 'sending %s' % repr(data)
332 self.socket.sendall((data + '\n').encode('utf-8'))
333 # }}}
334
346 def recv(self, maxsize = 4096):
347 if self.socket is None:
348 log('recv on closed socket')
349 raise EOFError('recv on closed socket')
350 ret = b''
351 try:
352 ret = self.socket.recv(maxsize)
353 if hasattr(self.socket, 'pending'):
354 while self.socket.pending():
355 ret += self.socket.recv(maxsize)
356 except:
357 log('Error reading from socket: %s' % sys.exc_info()[1])
358 self.close()
359 return ret
360 if len(ret) == 0:
361 ret = self.close()
362 if not self._disconnect_cb:
363 raise EOFError('network connection closed')
364 return ret
365 # }}}
366
374 def rawread(self, callback, error = None):
375 if self.socket is None:
376 return b''
377 ret = self.unread()
378 self._callback = (callback, None)
379 self._event = add_read(self.socket, callback, error)
380 return ret
381 # }}}
382
393 def read(self, callback, error = None, maxsize = 4096):
394 if self.socket is None:
395 return b''
396 first = self.unread()
397 self._maxsize = maxsize
398 self._callback = (callback, False)
399 def cb():
400 data = self.recv(self._maxsize)
401 #log('network read %d bytes' % len(data))
402 if not self._event:
403 return False
404 callback(data)
405 return True
406 self._event = add_read(self.socket, cb, error)
407 if first:
408 callback(first)
409 # }}}
410
422 def readlines(self, callback, error = None, maxsize = 4096):
423 if self.socket is None:
424 return
425 self._linebuffer = self.unread()
426 self._maxsize = maxsize
427 self._callback = (callback, True)
428 self._event = add_read(self.socket, self._line_cb, error)
429 # }}}
430 def _line_cb(self):
431 self._linebuffer += self.recv(self._maxsize)
432 while b'\n' in self._linebuffer and self._event:
433 assert self._callback[1] is not None # Going directly from readlines() to rawread() is not allowed.
434 if self._callback[1]:
435 line, self._linebuffer = self._linebuffer.split(b'\n', 1)
436 line = makestr(line)
437 self._callback[0] (line)
438 else:
439 data = makestr(self._linebuffer)
440 self._linebuffer = b''
441 self._callback[0](data)
442 return True
443 # }}}
444
449 def unread(self):
450 if self._event:
451 try:
452 remove_read(self._event)
453 except ValueError:
454 # The function already returned False.
455 pass
456 self._event = None
457 ret = self._linebuffer
458 self._linebuffer = b''
459 return ret
460 # }}}
461# }}}
462
463
464class Server:
465
494 def __init__(self, port, obj, address = '', backlog = 5, tls = False, disconnect_cb = None):
495 self._obj = obj
496
497 self.port = ''
498
499 self.ipv6 = False
500 self._socket = None
501
502 self.tls = tls
503
504 self.connections = set()
505
506 self.disconnect_cb = disconnect_cb
507 if isinstance(port, str) and '/' in port:
508 # Unix socket.
509 # TLS is ignored for these sockets.
510 self.tls = False
511 self._socket = socket.socket(socket.AF_UNIX)
512 self._socket.bind(port)
513 self.port = port
514 self._socket.listen(backlog)
515 else:
516 self._tls_init()
517 port = lookup(port)
518 self._socket = socket.socket()
519 self._socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
520 self._socket.bind((address, port))
521 self._socket.listen(backlog)
522 if address == '':
523 self._socket6 = socket.socket(socket.AF_INET6)
524 self._socket6.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
525 self._socket6.bind(('::1', port))
526 self._socket6.listen(backlog)
527 self.ipv6 = True
528 self.port = port
529 self._event = add_read(self._socket, lambda: self._cb(False), lambda: self._cb(False))
530 if self.ipv6:
531 self._event = add_read(self._socket6, lambda: self._cb(True), lambda: self._cb(True))
532 def _cb(self, is_ipv6):
533 if is_ipv6:
534 new_socket = self._socket6.accept()
535 else:
536 new_socket = self._socket.accept()
537 #log('Accepted connection from %s; possibly attempting to set up encryption' % repr(new_socket))
538 if self.tls:
539 assert have_ssl
540 try:
541 new_socket = (ssl.wrap_socket(new_socket[0], ssl_version = ssl.PROTOCOL_TLSv1, server_side = True, certfile = self._tls_cert, keyfile = self._tls_key), new_socket[1])
542 except ssl.SSLError as e:
543 log('Rejecting (non-TLS?) connection for %s: %s' % (repr(new_socket[1]), str(e)))
544 try:
545 new_socket[0].shutdown(socket.SHUT_RDWR)
546 except:
547 # Ignore errors here.
548 pass
549 return True
550 except socket.error as e:
551 log('Rejecting connection for %s: %s' % (repr(new_socket[1]), str(e)))
552 try:
553 new_socket[0].shutdown(socket.SHUT_RDWR)
554 except:
555 # Don't care about errors on shutdown.
556 pass
557 return True
558 #log('Accepted TLS connection from %s' % repr(new_socket[1]))
559 s = Socket(new_socket[0], remote = new_socket[1], disconnect_cb = self.disconnect_cb, connections = self.connections)
560 self._obj(s)
561 return True
562
565 def close(self):
566 self._socket.close()
567 self._socket = None
568 if self.ipv6:
569 self._socket6.close()
570 self._socket6 = None
571 if isinstance(self.port, str) and '/' in self.port:
572 os.remove(self.port)
573 self.port = ''
574
577 def __del__(self):
578 if self._socket is not None:
579 self.close()
580 def _tls_init(self):
581 # Set up members for using tls, if requested.
582 if self.tls in (False, '-'):
583 self.tls = False
584 return
585 if self.tls in (None, True, ''):
586 self.tls = fhs.module_get_config('network')['tls']
587 if self.tls == '':
588 self.tls = socket.getfqdn()
589 elif self.tls == '-':
590 self.tls = False
591 return
592 # Use tls.
593 fc = fhs.read_data(os.path.join('certs', self.tls + os.extsep + 'pem'), opened = False, packagename = 'network')
594 fk = fhs.read_data(os.path.join('private', self.tls + os.extsep + 'key'), opened = False, packagename = 'network')
595 if fc is None or fk is None:
596 # Create new self-signed certificate.
597 certfile = fhs.write_data(os.path.join('certs', self.tls + os.extsep + 'pem'), opened = False, packagename = 'network')
598 csrfile = fhs.write_data(os.path.join('csr', self.tls + os.extsep + 'csr'), opened = False, packagename = 'network')
599 for p in (certfile, csrfile):
600 path = os.path.dirname(p)
601 if not os.path.exists(path):
602 os.makedirs(path)
603 keyfile = fhs.write_data(os.path.join('private', self.tls + os.extsep + 'key'), opened = False, packagename = 'network')
604 path = os.path.dirname(keyfile)
605 if not os.path.exists(path):
606 os.makedirs(path, 0o700)
607 os.system('openssl req -x509 -nodes -days 3650 -newkey rsa:4096 -subj "/CN=%s" -keyout "%s" -out "%s"' % (self.tls, keyfile, certfile))
608 os.system('openssl req -subj "/CN=%s" -new -key "%s" -out "%s"' % (self.tls, keyfile, csrfile))
609 fc = fhs.read_data(os.path.join('certs', self.tls + os.extsep + 'pem'), opened = False, packagename = 'network')
610 fk = fhs.read_data(os.path.join('private', self.tls + os.extsep + 'key'), opened = False, packagename = 'network')
611 self._tls_cert = fc
612 self._tls_key = fk
613 #print(fc, fk)
614# }}}
615
616
619_timeouts = []
620
623_abort = False
624def _handle_timeouts():
625 now = time.time()
626 while not _abort and len(_timeouts) > 0 and _timeouts[0][0] <= now:
627 _timeouts.pop(0)[1]()
628 if len(_timeouts) == 0:
629 return float('inf')
630 return _timeouts[0][0] - now
631# }}}
632
633
636_fds = [[], []]
637
639def iteration(block = False):
640 # The documentation says timeout should be omitted, it doesn't mention making it None.
641 t = _handle_timeouts()
642 if not block:
643 t = 0
644 #log('do select with timeout %f' % t)
645 if math.isinf(t):
646 ret = select.select(_fds[0], _fds[1], _fds[0] + _fds[1])
647 else:
648 ret = select.select(_fds[0], _fds[1], _fds[0] + _fds[1], t)
649 #log('select returned %s' % repr(ret))
650 for f in ret[2]:
651 if f not in _fds[0] and f not in _fds[1]:
652 continue
653 if not f.error():
654 try:
655 remove_read(f)
656 except ValueError:
657 # The connection was already closed.
658 pass
659 if _abort:
660 return
661 for f in ret[0]:
662 if f not in _fds[0]:
663 continue
664 if not f.handle():
665 try:
666 remove_read(f)
667 except ValueError:
668 # The connection was already closed.
669 pass
670 if _abort:
671 return
672 for f in ret[1]:
673 if f not in _fds[1]:
674 continue
675 if not f.handle():
676 remove_write(f)
677 if _abort:
678 return
679 _handle_timeouts()
680# }}}
681
682
685_running = False
686
689_idle = []
690
694def fgloop():
695 global _running
696 assert not _running
697
700 _running = True
701 try:
702 while _running:
703 iteration(len(_idle) == 0)
704 if not _running:
705 return False
706 for i in _idle[:]:
707 if not i():
708 remove_idle(i)
709 if not _running:
710 break
711 finally:
712
715 _running = False
716
719 _abort = False
720 return False
721# }}}
722
723
728def bgloop():
729 assert _running == False
730 if os.getenv('NETWORK_NO_FORK') is None:
731 if os.fork() != 0:
732 sys.exit(0)
733 else:
734 log('Not backgrounding because NETWORK_NO_FORK is set\n')
735 fgloop()
736# }}}
737
738
741def endloop(force = False):
742 global _running, _abort
743 assert _running
744
747 _running = False
748 if force:
749
752 _abort = True
753# }}}
754
755class _fd_wrap:
756 def __init__(self, fd, cb, error):
757 self.fd = fd
758 self.handle = cb
759 if error is not None:
760 self.error = error
761 else:
762 self.error = self.default_error
763 def fileno(self):
764 if isinstance(self.fd, int):
765 return self.fd
766 else:
767 return self.fd.fileno()
768 def default_error(self):
769 try:
770 remove_read(self)
771 log('Error returned from select; removed fd from read list')
772 except:
773 try:
774 remove_write(self)
775 log('Error returned from select; removed fd from write list')
776 except:
777 log('Error returned from select, but fd was not in read or write list')
778# }}}
779
780def add_read(fd, cb, error = None):
781 _fds[0].append(_fd_wrap(fd, cb, error))
782 #log('add read %s' % repr(_fds[0][-1]))
783 return _fds[0][-1]
784# }}}
785
786def add_write(fd, cb, error = None):
787 _fds[1].append(_fd_wrap(fd, cb, error))
788 return _fds[1][-1]
789# }}}
790
791def add_timeout(abstime, cb):
792 _timeouts.append([abstime, cb])
793 ret = _timeouts[-1]
794 _timeouts.sort()
795 return ret
796# }}}
797
798def add_idle(cb):
799 _idle.append(cb)
800 return _idle[-1]
801# }}}
802
803def remove_read(handle):
804 #log('remove read %s' % repr(handle))
805 #traceback.print_stack()
806 _fds[0].remove(handle)
807# }}}
808
809def remove_write(handle):
810 _fds[1].remove(handle)
811# }}}
812
813def remove_timeout(handle):
814 _timeouts.remove(handle)
815# }}}
816
817def remove_idle(handle):
818 _idle.remove(handle)
819# }}}
tls
False or the hostname for which the TLS keys are used.
Definition network.py:502
ipv6
Whether the server listens for IPv6.
Definition network.py:499
port
Port that is listened on.
Definition network.py:497
disconnect_cb
Disconnect handler, to be used for new sockets.
Definition network.py:506
connections
Currently active connections for this server.
Definition network.py:504
close(self)
Stop the server.
Definition network.py:565
Listen on a network port and accept connections.
Definition network.py:464
recv(self, maxsize=4096)
Read data from the network.
Definition network.py:346
connections
connections set where this socket is registered.
Definition network.py:210
sendline(self, data)
Send a line of text.
Definition network.py:328
close(self)
Close the network connection.
Definition network.py:296
readlines(self, callback, error=None, maxsize=4096)
Buffer incoming data until a line is received, then call a function.
Definition network.py:422
send(self, data)
Send data over the network.
Definition network.py:313
remote
remote end of the network connection.
Definition network.py:208
rawread(self, callback, error=None)
Register function to be called when data is ready for reading.
Definition network.py:374
tls
read only variable which indicates whether TLS encryption is used on this socket.
Definition network.py:206
disconnect_cb(self, disconnect_cb)
Change the callback for disconnect notification.
Definition network.py:291
unread(self)
Cancel a read() or rawread() callback.
Definition network.py:449
socket
underlying socket object.
Definition network.py:214
read(self, callback, error=None, maxsize=4096)
Register function to be called when data is received.
Definition network.py:393
Connection object.
Definition network.py:187
remove_write(handle)
Definition network.py:809
bgloop()
Like fgloop, but forks to the background.
Definition network.py:728
add_timeout(abstime, cb)
Definition network.py:791
remove_timeout(handle)
Definition network.py:813
log(*message, filename=None, line=None, funcname=None, depth=0)
Log a message.
Definition network.py:102
add_read(fd, cb, error=None)
Definition network.py:780
add_idle(cb)
Definition network.py:798
lookup(service)
Convert int or str with int or service to int port.
Definition network.py:123
remove_idle(handle)
Definition network.py:817
wrap(i, o=None)
Wrap two files into a fake socket.
Definition network.py:181
set_log_output(file)
Change target for log().
Definition network.py:85
iteration(block=False)
Do a single iteration of the main loop.
Definition network.py:639
remove_read(handle)
Definition network.py:803
str makestr
Definition network.py:72
add_write(fd, cb, error=None)
Definition network.py:786
endloop(force=False)
Stop a loop that was started with fgloop() or bgloop().
Definition network.py:741
fgloop()
Wait for events and handle them.
Definition network.py:694