479 lines
15 KiB
Python
479 lines
15 KiB
Python
#!/usr/bin/env python3
|
|
# -*- coding: utf-8 -*-
|
|
|
|
__version__ = "0.0.6.2"
|
|
|
|
try:
|
|
import ConfigParser as cp
|
|
except:
|
|
import configparser as cp # python3
|
|
import optparse as op
|
|
import re
|
|
import os
|
|
import sys
|
|
import json
|
|
import subprocess
|
|
|
|
import stem
|
|
from stem.control import Controller
|
|
|
|
# Usage:
|
|
#
|
|
# Make a directory.
|
|
#
|
|
# Put a configuration file repo.cfg listing some peers. Done.
|
|
#
|
|
# Initialize:
|
|
# Either a) (git init repo/) ->
|
|
# $ python Globalist.py -i
|
|
# or b) (torsocks git clone git://example7abcdefgh.onion) ->
|
|
# $ python Globalist.py -c
|
|
#
|
|
# Have fun:
|
|
# Run server
|
|
# $ python Globalist.py
|
|
# Pull from peers once
|
|
# $ python Globalist.py -p
|
|
# Periodically pull, don't serve
|
|
# $ python Globalist.py -pP 1800
|
|
# Periodically pull and also serve
|
|
# $ python Globalist.py -P 1800
|
|
#
|
|
# That's it.
|
|
|
|
# One can simply check in a list of onions for open peering
|
|
# as PEERS.txt ...
|
|
|
|
# A word of CAUTION: anyone can commit anything
|
|
# and there's no mechanism for permanently blacklisting
|
|
# malicious peers (although one can simply remove them
|
|
# as they crop up and roll back their changes).
|
|
#
|
|
# A future version of Globalist.py should introduce
|
|
# signed commits + reputation system, when the need arises.
|
|
|
|
# [network]
|
|
# peers = example7abcdefgh.onion, example8abcdefgh.onion
|
|
# (possibly prefixed with somebody:authkey@ ...)
|
|
|
|
# when using -b (bare), merge remote changes locally after
|
|
# git pull origin remote/origin/master.
|
|
|
|
DEFAULT_CONTROLPORT = 9151
|
|
|
|
STATUS = {'peers': None, 'socksport': None}
|
|
|
|
OPTIONS = None
|
|
|
|
def git(command):
|
|
# print (command)
|
|
p = subprocess.Popen(["git"] + command)
|
|
return p
|
|
|
|
def make_exportable(path):
|
|
subprocess.Popen(["touch", os.path.abspath(os.path.join(path, "git-daemon-export-ok")) ]).wait()
|
|
|
|
def run_server(config, localport = 9418):
|
|
print ("Running git server on %s.onion" % config.get('onion', 'hostname'))
|
|
try:
|
|
authkey = config.get('onion', 'clientauth')
|
|
if authkey:
|
|
print ("Client auth is %s" % authkey)
|
|
except (KeyError, cp.NoOptionError) as e:
|
|
print ("No client auth")
|
|
print ("Git server local port is %d" % localport)
|
|
print ("You can now hand out this onion to prospective peers.")
|
|
print ("It will be re-used anytime Globalist starts in this directory.")
|
|
|
|
what = "repo"
|
|
|
|
if OPTIONS.o_bare:
|
|
make_exportable("repo.git")
|
|
what += ".git"
|
|
else:
|
|
make_exportable(os.path.join("repo",".git"))
|
|
|
|
gitdaemon = git(["daemon", "--base-path=%s" % os.path.abspath("."),
|
|
"--reuseaddr", "--verbose",
|
|
# there could be a global setting enabling write access??
|
|
"--disable=receive-pack",
|
|
"--listen=127.0.0.1", "--port=%d" % localport,
|
|
os.path.abspath(what)])
|
|
output = gitdaemon.communicate()[0]
|
|
print (output)
|
|
# then background this process
|
|
|
|
def makeonion(controller, config, options):
|
|
# stem docs say: provide the password here if you set one:
|
|
controller.authenticate()
|
|
# todo catch UnreadableCookieFile(
|
|
|
|
onion = None
|
|
|
|
extra_kwargs = {}
|
|
|
|
if config.has_section('onion'):
|
|
print ("Attempting to use saved onion identity")
|
|
(keytype,key) = config.get('onion', 'key').split(':',1)
|
|
|
|
if options.o_auth:
|
|
try:
|
|
print ("Attempting to use saved clientauth")
|
|
extra_kwargs['basic_auth'] =\
|
|
dict([config.get('onion', 'clientauth').split(':',1)])
|
|
except (KeyError, cp.NoOptionError) as e:
|
|
print ("No client auth present, generating one")
|
|
extra_kwargs['basic_auth'] = {'somebody': None}
|
|
else:
|
|
print ("Not using clientauth.")
|
|
|
|
onion = controller.create_ephemeral_hidden_service(**extra_kwargs, ports={9418: options.a_localport}, discard_key=True, await_publication=options.o_ap, key_type=keytype, key_content=key)
|
|
|
|
else:
|
|
print ("I'm afraid we don't have an identity yet, creating one")
|
|
|
|
if options.o_auth:
|
|
extra_kwargs['basic_auth'] = {'somebody': None}
|
|
|
|
onion = controller.create_ephemeral_hidden_service(**extra_kwargs, ports={9418: options.a_localport}, discard_key=False, await_publication=options.o_ap)
|
|
|
|
# print (onion)
|
|
|
|
print ("Tor controller says Onion OK")
|
|
|
|
if not onion.is_ok():
|
|
raise Exception('Failed to publish onion.')
|
|
else:
|
|
# perhaps avoid overwriting when already present?
|
|
for line in onion:
|
|
if line != "OK":
|
|
k, v = line.split('=', 1)
|
|
# we only request the key if the service is new
|
|
if k == "PrivateKey":
|
|
try:
|
|
config.add_section('onion')
|
|
except cp.DuplicateSectionError as e:
|
|
pass
|
|
config.set('onion', 'key', v)
|
|
if k == "ServiceID":
|
|
try:
|
|
config.add_section('onion')
|
|
except cp.DuplicateSectionError as e:
|
|
pass
|
|
config.set('onion', 'hostname', v)
|
|
if k == "ClientAuth":
|
|
try:
|
|
config.add_section('onion')
|
|
except cp.DuplicateSectionError as e:
|
|
pass
|
|
config.set('onion', 'clientauth', v)
|
|
config.write(open('repo.cfg', 'w'))
|
|
|
|
|
|
def set_client_authentications(ls):
|
|
global OPTIONS
|
|
options = OPTIONS
|
|
|
|
controller = Controller.from_port(port = options.a_controlport)
|
|
controller.authenticate()
|
|
# is there no sane way to _append_ a multi-config option in Tor????
|
|
# control protocol badly misdesigned, nobody thought of concurrent access???!?
|
|
controller.set_caching(False)
|
|
|
|
# except it doesn't work, the 650 message never arrives. why?
|
|
# controller.add_event_listener(my_confchanged_listener, EventType.CONF_CHANGED)
|
|
# SETEVENTS conf_changed
|
|
|
|
hsa = controller.get_conf_map('hidservauth')
|
|
|
|
for authpair in ls:
|
|
if authpair['auth'] and len(authpair['auth']):
|
|
hsa['hidservauth'].append('%s.onion %s' % (authpair['onion'], authpair['auth']))
|
|
|
|
hsa['hidservauth'] = list(set(hsa['hidservauth']))
|
|
|
|
controller.set_conf('hidservauth', hsa['hidservauth'])
|
|
controller.close()
|
|
|
|
|
|
def getpeers(config):
|
|
if STATUS['peers']:
|
|
return STATUS['peers']
|
|
|
|
if config.has_section('network'):
|
|
peerslist = config.get('network', 'peers').split(',')
|
|
peers = []
|
|
authpairs = []
|
|
|
|
for peerentry in peerslist:
|
|
|
|
# extract what looks like an onion identifier
|
|
try:
|
|
authpair = re.findall('(?:(somebody:[A-Za-z0-9+/]{22})@)?([a-z2-8]{16})', peerentry)[0]
|
|
|
|
userpass = authpair[0].split(":",1)
|
|
if not userpass or not len(userpass)==2:
|
|
userpass = (None, None)
|
|
|
|
authpairs += [{'auth':userpass[1],
|
|
'user':userpass[0], # somebody
|
|
'onion':authpair[1]}]
|
|
peers += [authpair[1]]
|
|
|
|
except Exception as e:
|
|
print (e)
|
|
|
|
set_client_authentications(authpairs)
|
|
|
|
STATUS['peers'] = peers
|
|
|
|
return peers
|
|
|
|
else:
|
|
STATUS['peers'] = []
|
|
|
|
return []
|
|
|
|
def clone(config):
|
|
peers = getpeers(config)
|
|
|
|
# FIXME: when the first fails, we should move on to the next..
|
|
|
|
what = "git://%s.onion/repo" % peers[0]
|
|
where = "repo"
|
|
how = []
|
|
|
|
if OPTIONS.o_bare:
|
|
what += ".git"
|
|
where += ".git"
|
|
how = ["--bare", "--mirror"]
|
|
|
|
cloneproc = subprocess.Popen(["torsocks", "-P", str(STATUS['socksport']), "git", "clone"] + how + [what, where])
|
|
if cloneproc.wait() != 0:
|
|
print ("Error cloning, exiting.")
|
|
return -1
|
|
else:
|
|
make_exportable(where)
|
|
|
|
# Make a local editable repo
|
|
try:
|
|
git(["clone", "repo", "repo.git"]).wait()
|
|
except:
|
|
print ("Failed to export repository, try to remove 'repo.git'.")
|
|
|
|
processes = []
|
|
for peer in peers[1:]:
|
|
processes.append([peer, subprocess.Popen(["torsocks", "-P", STATUS['socksport'], "git", "-C", os.path.abspath("repo"), "pull", "git://%s.onion/repo" % peer])])
|
|
for (peer,proc) in processes:
|
|
if proc.wait() != 0:
|
|
print ("Error with %s" % peer)
|
|
|
|
def pull(config):
|
|
peers = getpeers(config)
|
|
|
|
print ("Pulling from %s" % peers)
|
|
|
|
processes = []
|
|
for peer in peers:
|
|
processes.append([peer, subprocess.Popen(["torsocks", "-P", STATUS['socksport'], "git", "-C", os.path.abspath("repo"), "pull", "git://%s.onion/repo" % peer])])
|
|
for (peer,proc) in processes:
|
|
if proc.wait() != 0:
|
|
print ("Error with %s" % peer)
|
|
|
|
def fetch(config):
|
|
peers = getpeers(config)
|
|
print ("Fetching from %s" % peers)
|
|
processes = []
|
|
for peer in peers:
|
|
processes.append([peer, subprocess.Popen(["torsocks", "-P", STATUS['socksport'], "git", "-C", os.path.abspath("repo.git"), "fetch", "git://%s.onion/repo.git" % peer, '+refs/heads/*:refs/remotes/origin/*'])])
|
|
|
|
for (peer,proc) in processes:
|
|
if proc.wait() != 0:
|
|
print ("Error with %s" % peer)
|
|
|
|
def init(config):
|
|
global OPTIONS # not needed for read access btw
|
|
options = OPTIONS
|
|
|
|
print ("Initializing ...")
|
|
|
|
if options.o_bare:
|
|
git(["init", "repo.git", "--bare"]).wait()
|
|
# Make a local editable repo
|
|
git(["clone", "repo.git", "repo"]).wait()
|
|
|
|
else:
|
|
git(["init", "repo"]).wait()
|
|
|
|
print ("Initialized")
|
|
|
|
def main(args=[]):
|
|
# OptionParser is capable of printing a helpscreen
|
|
opt = op.OptionParser()
|
|
|
|
opt.add_option("-V", "--version", dest="o_version", action="store_true",
|
|
default=False, help="print version number")
|
|
|
|
opt.add_option("-i", "--init", dest="o_init", action="store_true",
|
|
default=False, help="make new empty repo")
|
|
|
|
opt.add_option("-b", "--bare", dest="o_bare", action="store_true",
|
|
default=False, help="use bare repos and fetch, not pull")
|
|
|
|
opt.add_option("-c", "--clone", dest="o_clone", action="store_true",
|
|
default=False, help="clone repo from 1st peer")
|
|
|
|
opt.add_option("-p", "--pull", dest="o_pull", action="store_true",
|
|
default=False, help="pull / fetch from peers and don't serve")
|
|
|
|
opt.add_option("-P", "--periodically-pull", dest="a_pull", action="store",
|
|
type="int", default=None, metavar="PERIOD",
|
|
help="pull / fetch from peers every n seconds")
|
|
|
|
opt.add_option("-L", "--local", dest="a_localport", action="store", type="int",
|
|
default=9418, metavar="PORT", help="local port for git daemon")
|
|
|
|
opt.add_option("-C", "--control-port", dest="a_controlport", action="store", type="int",
|
|
default=9151, metavar="PORT", help="Tor controlport")
|
|
|
|
# opt.add_option("-CP", "--control-password", dest="a_controlpassword", action="store", type="int",
|
|
# default="", help="Tor Control Password")
|
|
|
|
# opt.add_option("-CC", "--control-cookie", dest="a_controlcookie", action="store", type="int",
|
|
# default="", help="Tor Control Cookie")
|
|
|
|
opt.add_option("-a", "--await", dest="o_ap", action="store_true",
|
|
default=False, help="await publication of .onion in DHT before proceeding")
|
|
|
|
opt.add_option("-x", "--auth", action="store_true", default=True,
|
|
dest="o_auth", help="enable authentication (private)")
|
|
|
|
opt.add_option("-X", "--no-auth", action="store_false", default=True,
|
|
dest="o_auth", help="disable authentication (not private)")
|
|
|
|
(options, args) = opt.parse_args(args)
|
|
|
|
global OPTIONS
|
|
OPTIONS = options
|
|
|
|
if options.o_version:
|
|
print (__version__)
|
|
return 0
|
|
|
|
if options.o_auth and stem.__version__ < '1.5.0':
|
|
sys.stderr.write ("stem version >=1.5.0 required for auth\n")
|
|
return 1
|
|
|
|
if not options.a_controlport:
|
|
options.a_controlport = DEFAULT_CONTROLPORT
|
|
|
|
# Extract socksport via c.get_conf and use this (-P in torsocks)
|
|
# TODO implement authentication token / cookie
|
|
controller = Controller.from_port(port = options.a_controlport)
|
|
controller.authenticate()
|
|
if controller.get_conf('SocksPort'):
|
|
STATUS['socksport'] = controller.get_conf('SocksPort').split(" ",1)[0]
|
|
else:
|
|
STATUS['socksport'] = 9050
|
|
controller.close()
|
|
|
|
config = cp.ConfigParser()
|
|
cfgfile = None
|
|
try:
|
|
cfgfile = open('repo.cfg')
|
|
except FileNotFoundError as e:
|
|
print("Trying to make file repo.cfg")
|
|
try:
|
|
os.mknod("repo.cfg")
|
|
os.chmod("repo.cfg", 0o600)
|
|
cfgfile = open('repo.cfg')
|
|
except Exception as e:
|
|
print (e)
|
|
return 1
|
|
|
|
config.readfp(cfgfile)
|
|
|
|
try:
|
|
os.stat("repo.git")
|
|
if not options.o_bare:
|
|
print ("repo.git exists, setting -b implicitly")
|
|
# TODO -B to override
|
|
options.o_bare = True
|
|
|
|
except FileNotFoundError as e:
|
|
if not options.o_init and not options.o_clone and options.o_bare:
|
|
print ("./repo.git/ does not exist, try -ib or -cb")
|
|
return 1
|
|
|
|
try:
|
|
os.stat("repo")
|
|
except FileNotFoundError as e:
|
|
if not options.o_init and not options.o_clone and not options.o_bare:
|
|
print("./repo/ does not exist, try -i or -c")
|
|
return 1
|
|
|
|
except Exception as e:
|
|
print (e)
|
|
return 1
|
|
|
|
if options.o_init:
|
|
init(config)
|
|
|
|
peers = getpeers(config)
|
|
|
|
if options.o_clone:
|
|
if not len(peers):
|
|
print ("No peers, can't clone. Please enter a peer in repo.cfg")
|
|
clone(config)
|
|
return 1
|
|
|
|
threads = []
|
|
|
|
if options.a_pull:
|
|
if not len(peers):
|
|
print ("No peers, not starting pulling task.")
|
|
|
|
else:
|
|
import threading
|
|
from datetime import timedelta as td
|
|
from datetime import datetime
|
|
|
|
class T:
|
|
def __init__(self):
|
|
self.last = datetime.now()
|
|
|
|
def run(self):
|
|
if options.o_bare:
|
|
fetch(config)
|
|
else:
|
|
pull(config)
|
|
threading.Timer(options.a_pull, T.run, args=(self,)).start()
|
|
|
|
task = T()
|
|
|
|
t = threading.Thread(target=T.run, args=(task,))
|
|
t . setDaemon(True)
|
|
threads.append(t)
|
|
t.start()
|
|
|
|
# It's either pull(once) or serve. It's no problem running pull from
|
|
# another console while the server is up. It's no problem specifying
|
|
# periodic pull with either.
|
|
|
|
if options.o_pull and not options.a_pull:
|
|
if options.o_bare:
|
|
fetch(config)
|
|
else:
|
|
pull(config)
|
|
|
|
elif not options.o_pull:
|
|
controller = Controller.from_port(port = options.a_controlport)
|
|
makeonion(controller, config, options)
|
|
run_server(config, localport = options.a_localport)
|
|
controller.close()
|
|
|
|
for t in threads:
|
|
t.join()
|
|
|
|
# TODO: should only generate a clientauth on a previously unauthenticated repo if requested by command line option
|