waybackproxy.py 24 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602
  1. #!/usr/bin/env python3
  2. import base64, datetime, json, lrudict, re, socket, socketserver, string, sys, threading, traceback, urllib.request, urllib.error, urllib.parse
  3. from config_handler import *
  4. class ThreadingTCPServer(socketserver.ThreadingMixIn, socketserver.TCPServer):
  5. """TCPServer with ThreadingMixIn added."""
  6. pass
  7. class SharedState:
  8. """Class for storing shared state across instances of Handler."""
  9. def __init__(self):
  10. # Create internal LRU dictionary for preserving URLs on redirect.
  11. self.date_cache = lrudict.LRUDict(maxduration=86400, maxsize=1024)
  12. # Create internal LRU dictionary for date availability.
  13. self.availability_cache = lrudict.LRUDict(maxduration=86400, maxsize=1024) if WAYBACK_API else None
  14. # Read domain whitelist file.
  15. try:
  16. with open('whitelist.txt', 'r') as f:
  17. self.whitelist = f.read().splitlines()
  18. except:
  19. self.whitelist = []
  20. shared_state = SharedState()
  21. class Handler(socketserver.BaseRequestHandler):
  22. """Main request handler."""
  23. def setup(self, *args, **kwargs):
  24. """Set up this instance of Handler."""
  25. super().setup(*args, **kwargs)
  26. # Store a local pointer to SharedState.
  27. self.shared_state = shared_state
  28. def handle(self):
  29. """Handle a request."""
  30. # readline is pretty convenient
  31. f = self.request.makefile()
  32. # read request line
  33. reqline = line = f.readline()
  34. split = line.rstrip().split(' ')
  35. http_version = len(split) > 2 and split[2] or 'HTTP/0.9'
  36. if len(split) < 2 or split[0] != 'GET':
  37. # only GET is implemented
  38. return self.send_error_page(http_version, 501, 'Not Implemented')
  39. # read out the headers
  40. request_host = None
  41. pac_host = '" + location.host + ":' + str(LISTEN_PORT) # may not actually work
  42. effective_date = DATE
  43. auth = None
  44. while line.strip() != '':
  45. line = f.readline()
  46. ll = line.lower()
  47. if ll[:6] == 'host: ':
  48. pac_host = request_host = line[6:].rstrip()
  49. if ':' not in pac_host: # explicitly specify port if running on port 80
  50. pac_host += ':80'
  51. elif ll[:21] == 'x-waybackproxy-date: ':
  52. # API for a personal project of mine
  53. effective_date = line[21:].rstrip()
  54. elif ll[:21] == 'authorization: basic ':
  55. # asset date code passed as username:password
  56. auth = base64.b64decode(ll[21:])
  57. # parse the URL
  58. pac_file_paths = ('/proxy.pac', '/wpad.dat', '/wpad.da')
  59. if split[1][0] == '/' and split[1] not in pac_file_paths:
  60. # just a path (not corresponding to a PAC file) => transparent proxy
  61. # Host header and therefore HTTP/1.1 are required
  62. if not request_host:
  63. return self.send_error_page(http_version, 400, 'Host header missing')
  64. archived_url = 'http://' + request_host + split[1]
  65. else:
  66. # full URL => explicit proxy
  67. archived_url = split[1]
  68. request_url = archived_url
  69. parsed = urllib.parse.urlparse(request_url)
  70. # make a path
  71. path = parsed.path
  72. if parsed.query:
  73. path += '?' + parsed.query
  74. elif path == '':
  75. path == '/'
  76. # get the hostname for later
  77. host = parsed.netloc.split(':')
  78. hostname = host[0]
  79. # get cached date for redirects, if available
  80. original_date = effective_date
  81. effective_date = self.shared_state.date_cache.get(str(effective_date) + '\x00' + str(archived_url), effective_date)
  82. # get date from username:password, if available
  83. if auth:
  84. effective_date = auth.replace(':', '')
  85. # Effectively handle the request.
  86. try:
  87. if path in pac_file_paths:
  88. # PAC file to bypass QUICK_IMAGES requests if WAYBACK_API is not enabled.
  89. pac = http_version + ''' 200 OK\r\n'''
  90. pac += '''Content-Type: application/x-ns-proxy-autoconfig\r\n'''
  91. pac += '''\r\n'''
  92. pac += '''function FindProxyForURL(url, host)\r\n'''
  93. pac += '''{\r\n'''
  94. if self.shared_state.availability_cache == None:
  95. pac += ''' if (shExpMatch(url, "http://web.archive.org/web/*") && !shExpMatch(url, "http://web.archive.org/web/??????????????if_/*"))\r\n'''
  96. pac += ''' {\r\n'''
  97. pac += ''' return "DIRECT";\r\n'''
  98. pac += ''' }\r\n'''
  99. pac += ''' return "PROXY ''' + pac_host + '''";\r\n'''
  100. pac += '''}\r\n'''
  101. self.request.sendall(pac.encode('ascii', 'ignore'))
  102. return
  103. elif hostname in self.shared_state.whitelist:
  104. _print('[>] [byp]', archived_url)
  105. elif hostname == 'web.archive.org':
  106. if path[:5] != '/web/':
  107. # Launch settings if enabled.
  108. if SETTINGS_PAGE:
  109. return self.handle_settings(parsed.query)
  110. else:
  111. return self.send_error_page(http_version, 404, 'Not Found')
  112. else:
  113. # Pass requests through to web.archive.org. Required for QUICK_IMAGES.
  114. split = request_url.split('/')
  115. effective_date = split[4]
  116. archived_url = '/'.join(split[5:])
  117. _print('[>] [QI]', archived_url)
  118. elif GEOCITIES_FIX and hostname == 'www.geocities.com':
  119. # Apply GEOCITIES_FIX and pass it through.
  120. _print('[>]', archived_url)
  121. split = archived_url.split('/')
  122. hostname = split[2] = 'www.oocities.org'
  123. request_url = '/'.join(split)
  124. else:
  125. # Get from the Wayback Machine.
  126. _print('[>]', archived_url)
  127. request_url = 'http://web.archive.org/web/{0}if_/{1}'.format(effective_date, archived_url)
  128. # Check Wayback Machine Availability API where applicable, to avoid archived 404 pages and other site errors.
  129. if self.shared_state.availability_cache != None:
  130. # Are we requesting from the Wayback Machine?
  131. split = request_url.split('/')
  132. # If so, get the closest available date from the API.
  133. if split[2] == 'web.archive.org':
  134. # Remove extraneous :80 from URL.
  135. if ':' in split[5]:
  136. if split[7][-3:] == ':80':
  137. split[7] = split[7][:-3]
  138. elif split[5][-3:] == ':80':
  139. split[5] = split[5][:-3]
  140. # Check availability LRU cache.
  141. availability_url = '/'.join(split[5:])
  142. new_url = self.shared_state.availability_cache.get(availability_url, None)
  143. if new_url:
  144. # In cache => replace URL immediately.
  145. request_url = new_url
  146. else:
  147. # Not in cache => contact API.
  148. try:
  149. availability = json.loads(urllib.request.urlopen('https://archive.org/wayback/available?url=' + urllib.parse.quote_plus(availability_url) + '&timestamp=' + effective_date[:14], timeout=10).read())
  150. closest = availability.get('archived_snapshots', {}).get('closest', {})
  151. new_date = closest.get('timestamp', None)
  152. except:
  153. _print('[!] Failed to fetch Wayback availability data')
  154. new_date = None
  155. if new_date and new_date != effective_date[:14]:
  156. # Returned date is different.
  157. new_url = closest['url']
  158. # Add asset tag to the date.
  159. split = new_url.split('/')
  160. if len(effective_date) > 14:
  161. split[4] += effective_date[14:]
  162. else:
  163. split[4] += 'if_'
  164. new_url = '/'.join(split)
  165. # Replace URL and add it to the availability cache.
  166. request_url = self.shared_state.availability_cache[availability_url] = new_url
  167. # Start fetching the URL.
  168. conn = urllib.request.urlopen(request_url)
  169. except urllib.error.HTTPError as e:
  170. # An HTTP error has occurred.
  171. if e.code in (403, 404): # not found
  172. if self.guess_and_send_redirect(http_version, archived_url):
  173. return
  174. elif e.code in (301, 302): # urllib-generated error about an infinite redirect loop
  175. _print('[!] Infinite redirect loop')
  176. return self.send_error_page(http_version, 508, 'Infinite Redirect Loop')
  177. if e.code != 412: # tolerance exceeded has its own error message above
  178. _print('[!]', e.code, e.reason)
  179. # If the memento Link header is present, this is a website error
  180. # instead of a Wayback error. Pass it along if that's the case.
  181. if 'Link' in e.headers:
  182. conn = e
  183. else:
  184. return self.send_error_page(http_version, e.code, e.reason)
  185. except socket.timeout as e:
  186. # A timeout has occurred.
  187. _print('[!] Fetch timeout')
  188. return self.send_error_page(http_version, 504, 'Gateway Timeout')
  189. except:
  190. # Some other fetch exception has occurred.
  191. _print('[!] Fetch exception:')
  192. traceback.print_exc()
  193. return self.send_error_page(http_version, 502, 'Bad Gateway')
  194. # Get content type.
  195. content_type = conn.info().get('Content-Type')
  196. if content_type == None:
  197. content_type = 'text/html'
  198. elif not CONTENT_TYPE_ENCODING:
  199. idx = content_type.find(';')
  200. if idx > -1:
  201. content_type = content_type[:idx]
  202. # Set the archive mode.
  203. if GEOCITIES_FIX and hostname in ('www.oocities.org', 'www.oocities.com'):
  204. mode = 1 # oocities
  205. else:
  206. mode = 0 # Wayback Machine
  207. # Check content type to determine if this is HTML we need to patch.
  208. # Wayback will add its HTML to anything it thinks is HTML.
  209. guessed_content_type = conn.info().get('X-Archive-Guessed-Content-Type')
  210. if not guessed_content_type:
  211. guessed_content_type = content_type
  212. if 'text/html' in guessed_content_type:
  213. # Some dynamically-generated links may end up pointing to
  214. # web.archive.org. Correct that by redirecting the Wayback
  215. # portion of the URL away if it ends up being HTML consumed
  216. # through the QUICK_IMAGES interface.
  217. if hostname == 'web.archive.org':
  218. conn.close()
  219. archived_url = '/'.join(request_url.split('/')[5:])
  220. _print('[r] [QI]', archived_url)
  221. return self.send_redirect_page(http_version, archived_url, 301)
  222. # Check if the date is within tolerance.
  223. if DATE_TOLERANCE != None:
  224. match = re.search('''//web\\.archive\\.org/web/([0-9]+)''', conn.geturl())
  225. if match:
  226. requested_date = match.group(1)
  227. if self.wayback_to_datetime(requested_date) > self.wayback_to_datetime(original_date) + datetime.timedelta(int(DATE_TOLERANCE)):
  228. _print('[!]', requested_date, 'is outside the configured tolerance of', DATE_TOLERANCE, 'days')
  229. conn.close()
  230. if not self.guess_and_send_redirect(http_version, archived_url):
  231. self.send_error_page(http_version, 412, 'Snapshot ' + requested_date + ' not available')
  232. return
  233. # Consume all data.
  234. data = conn.read()
  235. # Patch the page.
  236. if mode == 0: # Wayback Machine
  237. # Check if this is a Wayback Machine page.
  238. if b'<title>Wayback Machine</title>' in data:
  239. # Check if this is an exclusion (robots.txt?) error page.
  240. if b'<p>This URL has been excluded from the Wayback Machine.</p>' in data:
  241. return self.send_error_page(http_version, 403, 'URL excluded')
  242. # Check if this is a media playback iframe page.
  243. # Some websites (especially ones that use frames)
  244. # inexplicably render inside a media playback iframe.
  245. # In that case, a simple redirect would result in a
  246. # redirect loop, so fetch and render the URL instead.
  247. match = re.search(b'''<iframe id="playback" src="((?:(?:https?:)?//web.archive.org)?/web/[^"]+)"''', data)
  248. if match:
  249. # Extract the content URL.
  250. request_url = match.group(1).decode('ascii', 'ignore')
  251. archived_url = '/'.join(request_url.split('/')[5:])
  252. # Start fetching the URL.
  253. _print('[f]', archived_url)
  254. try:
  255. conn = urllib.request.urlopen(request_url)
  256. except urllib.error.HTTPError as e:
  257. _print('[!]', e.code, e.reason)
  258. # If the memento Link header is present, this is a website error
  259. # instead of a Wayback error. Pass it along if that's the case.
  260. if 'Link' in e.headers:
  261. conn = e
  262. else:
  263. return self.send_error_page(http_version, e.code, e.reason)
  264. # Identify content type so we don't modify non-HTML content.
  265. content_type = conn.info().get('Content-Type')
  266. if not CONTENT_TYPE_ENCODING:
  267. idx = content_type.find(';')
  268. if idx > -1:
  269. content_type = content_type[:idx]
  270. if 'text/html' in content_type:
  271. # Consume all data and proceed with patching the page.
  272. data = conn.read()
  273. else:
  274. # Pass non-HTML data through.
  275. return self.send_passthrough(conn, http_version, content_type, request_url)
  276. # Check if this is a Wayback Machine redirect page.
  277. if b'<title></title>' in data and b'<span class="label style-scope media-button"><!---->Wayback Machine<!----></span>' in data:
  278. match = re.search(b'''<p class="impatient"><a href="(?:(?:https?:)?//web\\.archive\\.org)?/web/([^/]+)/([^"]+)">Impatient\\?</a></p>''', data)
  279. if match:
  280. # Sanitize the URL.
  281. archived_url = self.sanitize_redirect(match.group(2).decode('ascii', 'ignore'))
  282. # Add URL to the date LRU cache.
  283. self.shared_state.date_cache[str(effective_date) + '\x00' + archived_url] = match.group(1).decode('ascii', 'ignore')
  284. # Get the original HTTP redirect code.
  285. match = re.search(b'''<p class="code shift red">Got an HTTP ([0-9]+)''', data)
  286. try:
  287. redirect_code = int(match.group(1))
  288. except:
  289. redirect_code = 302
  290. # Redirect client to the URL.
  291. _print('[r]', archived_url)
  292. return self.send_redirect_page(http_version, archived_url, redirect_code)
  293. # Remove pre-toolbar scripts and CSS.
  294. data = re.sub(b'''<script (?:src="//archive\\.org/|type="text/javascript" src="/_static/js/).*<!-- End Wayback Rewrite JS Include -->\\r?\\n''', b'', data, count=1, flags=re.S)
  295. # Remove toolbar. The if_ asset tag serves no toolbar, but we remove it just in case.
  296. data = re.sub(b'''<!-- BEGIN WAYBACK TOOLBAR INSERT -->.*<!-- END WAYBACK TOOLBAR INSERT -->''', b'', data, count=1, flags=re.S)
  297. # Remove comments on footer.
  298. data = re.sub(b'''<!--\\r?\\n FILE ARCHIVED .*$''', b'', data, flags=re.S)
  299. # Fix base tag.
  300. data = re.sub(b'''(<base\\s+[^>]*href=["']?)(?:(?:https?:)?//web.archive.org)?/web/[^/]+/(?:[^:/]+://)?''', b'\\1http://', data, flags=re.I + re.S)
  301. # Remove extraneous :80 from links.
  302. data = re.sub(b'((?:(?:https?:)?//web.archive.org)?/web/)([^/]+)/([^/:]+)://([^/:]+):80/', b'\\1\\2/\\3://\\4/', data)
  303. # Fix links.
  304. if QUICK_IMAGES:
  305. # QUICK_IMAGES works by intercepting asset URLs (those
  306. # with a date code ending in im_, js_...) and letting the
  307. # proxy pass them through. This may reduce load time
  308. # because Wayback doesn't have to hunt down the closest
  309. # copy of that asset to DATE, as those URLs have specific
  310. # date codes. This taints the HTML with web.archive.org
  311. # URLs. QUICK_IMAGES=2 uses the original URLs with an added
  312. # username:password, which taints less but is not supported
  313. # by all browsers - IE notably kills the whole page if it
  314. # sees an iframe pointing to an invalid URL.
  315. data = re.sub(b'(?:(?:https?:)?//web.archive.org)?/web/([0-9]+)([a-z]+_)/([^:/]+://)',
  316. QUICK_IMAGES == 2 and b'\\3\\1:\\2@' or b'http://web.archive.org/web/\\1\\2/\\3', data)
  317. def strip_https(match): # convert secure non-asset URLs to regular HTTP
  318. first_component = match.group(1)
  319. return first_component == b'https:' and b'http:' or first_component
  320. data = re.sub(b'(?:(?:https?:)?//web.archive.org)?/web/[^/]+/([^/]+)', strip_https, data)
  321. else:
  322. # Remove asset URLs while simultaneously adding them to the date LRU cache
  323. # with their respective date and converting secure URLs to regular HTTP.
  324. def add_to_date_cache(match):
  325. orig_url = match.group(2)
  326. if orig_url[:8] == b'https://':
  327. orig_url = b'http://' + orig_url[8:]
  328. self.shared_state.date_cache[str(effective_date) + '\x00' + orig_url.decode('ascii', 'ignore')] = match.group(1).decode('ascii', 'ignore')
  329. return orig_url
  330. data = re.sub(b'''(?:(?:https?:)?//web.archive.org)?/web/([^/]+)/([^"\\'#<>]+)''', add_to_date_cache, data)
  331. elif mode == 1: # oocities
  332. # Remove viewport/cache-control/max-width code from the header.
  333. data = re.sub(b'''^.*?\n\n''', b'', data, flags=re.S)
  334. # Remove archive notice and tracking code from the footer.
  335. data = re.sub(b'''<style> \n.zoomout { -webkit-transition: .*$''', b'', data, flags=re.S)
  336. # Remove clearly labeled snippets from Geocities.
  337. data = re.sub(b'''^.*<\\!-- text above generated by server\\. PLEASE REMOVE -->''', b'', data, flags=re.S)
  338. data = re.sub(b'''<\\!-- following code added by server\\. PLEASE REMOVE -->.*<\!-- preceding code added by server\. PLEASE REMOVE -->''', b'', data, flags=re.S)
  339. data = re.sub(b'''<\\!-- text below generated by server\\. PLEASE REMOVE -->.*$''', b'', data, flags=re.S)
  340. # Fix links.
  341. data = re.sub(b'''//([^\\.]*\\.)?oocities\\.com/''', b'//\\1geocities.com/', data, flags=re.S)
  342. # Send patched page.
  343. self.send_response_headers(conn, http_version, content_type, request_url, content_length=len(data))
  344. self.request.sendall(data)
  345. self.request.close()
  346. else:
  347. # Pass non-HTML data through.
  348. self.send_passthrough(conn, http_version, content_type, request_url)
  349. def send_passthrough(self, conn, http_version, content_type, request_url):
  350. """Pass data through to the client unmodified (save for our headers)."""
  351. self.send_response_headers(conn, http_version, content_type, request_url, content_length=True)
  352. while True:
  353. data = conn.read(1024)
  354. if not data:
  355. break
  356. self.request.sendall(data)
  357. self.request.close()
  358. def send_response_headers(self, conn, http_version, content_type, request_url, content_length=False):
  359. """Generate and send the response headers."""
  360. # Pass the HTTP version, and error code if there is one.
  361. response = http_version
  362. if isinstance(conn, urllib.error.HTTPError):
  363. response += ' {0} {1}'.format(conn.code, conn.reason.replace('\n', ' '))
  364. else:
  365. response += ' 200 OK'
  366. # Add Content-Type, Content-Length and the caching ETag.
  367. response += '\r\nContent-Type: ' + content_type
  368. if type(content_length) == int:
  369. response += '\r\nContent-Length: ' + str(content_length)
  370. content_length = False # don't pass the original length through
  371. response += '\r\nETag: "' + request_url.replace('"', '') + '"'
  372. # Pass X-Archive-Orig-* (and Content-Length if requested) headers through.
  373. for header in conn.headers:
  374. if header.find('X-Archive-Orig-') == 0:
  375. orig_header = header[15:]
  376. # Skip headers which may affect client behavior.
  377. if orig_header.lower() not in ('connection', 'location', 'content-type', 'content-length', 'etag', 'authorization', 'set-cookie'):
  378. response += '\r\n' + orig_header + ': ' + conn.headers[header]
  379. elif content_length and header.lower() == 'content-length':
  380. response += '\r\n' + header + ': ' + conn.headers[header]
  381. # Finish and send the request.
  382. response += '\r\n\r\n'
  383. self.request.sendall(response.encode('utf8', 'ignore'))
  384. def send_error_page(self, http_version, code, reason):
  385. """Generate an error page."""
  386. # Get a description for this error code.
  387. if code in (404, 508): # page not archived or redirect loop
  388. description = 'This page may not be archived by the Wayback Machine.'
  389. elif code == 403: # not crawled due to exclusion
  390. description = 'This page was not archived due to a Wayback Machine exclusion.'
  391. elif code == 501: # method not implemented
  392. description = 'WaybackProxy only implements the GET method.'
  393. elif code == 502: # exception
  394. description = 'This page could not be fetched due to an unknown error.'
  395. elif code == 504: # timeout
  396. description = 'This page could not be fetched due to a Wayback Machine server timeout.'
  397. elif code == 412: # outside of tolerance
  398. description = 'The earliest snapshot for this page is outside of the configured tolerance interval.'
  399. elif code == 400 and reason == 'Host header missing': # no host header in transparent mode
  400. description = 'WaybackProxy\'s transparent mode requires an HTTP/1.1 compliant client.'
  401. else: # another error
  402. description = 'Unknown error. The Wayback Machine may be experiencing technical difficulties.'
  403. # Read error page file.
  404. try:
  405. f = open('error.html', 'r', encoding='utf8', errors='ignore')
  406. error_page = f.read()
  407. f.close()
  408. except:
  409. # Just send the code and reason as a backup.
  410. error_page = '${code} ${reason}'
  411. # Format error page template.
  412. signature = self.signature()
  413. error_page = string.Template(error_page).substitute(**locals())
  414. error_page_len = len(error_page)
  415. # Send formatted error page and stop.
  416. self.request.sendall(
  417. '{http_version} {code} {reason}\r\n'
  418. 'Content-Type: text/html\r\n'
  419. 'Content-Length: {error_page_len}\r\n'
  420. '\r\n'
  421. '{error_page}'
  422. .format(**locals()).encode('utf8', 'ignore')
  423. )
  424. self.request.close()
  425. def send_redirect_page(self, http_version, target, code=302):
  426. """Generate a redirect page."""
  427. # make redirect page
  428. redirectpage = '<html><head><title>Redirect</title><meta http-equiv="refresh" content="0;url='
  429. redirectpage += target
  430. redirectpage += '"></head><body><p>If you are not redirected, <a href="'
  431. redirectpage += target
  432. redirectpage += '">click here</a>.</p></body></html>'
  433. # send redirect page and stop
  434. self.request.sendall('{0} {1} Found\r\nLocation: {2}\r\nContent-Type: text/html\r\nContent-Length: {3}\r\n\r\n{4}'.format(http_version, code, target, len(redirectpage), redirectpage).encode('utf8', 'ignore'))
  435. self.request.close()
  436. def guess_and_send_redirect(self, http_version, guess_url):
  437. # Heuristically determine the static URL for some redirect scripts.
  438. parsed = urllib.parse.urlparse(guess_url)
  439. match = re.search('''(?:^|&)[^=]+=((?:https?(?:%3A|:)(?:%2F|/)|www[0-9]*\\.[^/%]+)?(?:%2F|/)[^&]+)''', parsed.query, re.I) # URL in query parameters
  440. if not match:
  441. full_path = parsed.path
  442. if parsed.query:
  443. full_path += '?' + parsed.query
  444. match = re.search('''((?:https?(?:%3A|:)(?:%2F|/)|www[0-9]*\\.[^/%]+)(?:%2F|/).+)''', full_path, re.I) # URL in path or full query
  445. if match: # found URL
  446. # Decode and sanitize the URL.
  447. new_url = self.sanitize_redirect(urllib.parse.unquote_plus(match.group(1)))
  448. # Redirect client to the URL.
  449. _print('[r] [g]', new_url)
  450. self.send_redirect_page(http_version, new_url)
  451. return True
  452. return False
  453. def handle_settings(self, query):
  454. """Generate the settings page."""
  455. global DATE, DATE_TOLERANCE, GEOCITIES_FIX, QUICK_IMAGES, WAYBACK_API, CONTENT_TYPE_ENCODING, SILENT, SETTINGS_PAGE
  456. if query != '': # handle any parameters that may have been sent
  457. parsed = urllib.parse.parse_qs(query)
  458. if 'date' in parsed and 'dateTolerance' in parsed:
  459. if DATE != parsed['date'][0]:
  460. DATE = parsed['date'][0]
  461. self.shared_state.date_cache.clear()
  462. self.shared_state.availability_cache.clear()
  463. if DATE_TOLERANCE != parsed['dateTolerance'][0]:
  464. DATE_TOLERANCE = parsed['dateTolerance'][0]
  465. GEOCITIES_FIX = 'gcFix' in parsed
  466. QUICK_IMAGES = 'quickImages' in parsed
  467. CONTENT_TYPE_ENCODING = 'ctEncoding' in parsed
  468. # send the page and stop
  469. settingspage = 'HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n\r\n'
  470. settingspage += '<html><head><title>WaybackProxy Settings</title></head><body><p><b>'
  471. settingspage += self.signature()
  472. settingspage += '</b></p><form method="get" action="/">'
  473. settingspage += '<p>Date to get pages from: <input type="text" name="date" size="8" value="'
  474. settingspage += str(DATE)
  475. settingspage += '"><p>Date tolerance: <input type="text" name="dateTolerance" size="8" value="'
  476. settingspage += str(DATE_TOLERANCE)
  477. settingspage += '"> days<br><input type="checkbox" name="gcFix"'
  478. if GEOCITIES_FIX:
  479. settingspage += ' checked'
  480. settingspage += '> Geocities Fix<br><input type="checkbox" name="quickImages"'
  481. if QUICK_IMAGES:
  482. settingspage += ' checked'
  483. settingspage += '> Quick images<br><input type="checkbox" name="ctEncoding"'
  484. if CONTENT_TYPE_ENCODING:
  485. settingspage += ' checked'
  486. settingspage += '> Encoding in Content-Type</p><p><input type="submit" value="Save"></p></form></body></html>'
  487. self.request.send(settingspage.encode('utf8', 'ignore'))
  488. self.request.close()
  489. def sanitize_redirect(self, url):
  490. """Sanitize an URL for client-side redirection."""
  491. if url[0] != '/' and '://' not in url:
  492. # Add protocol if the URL is absolute but missing a protocol.
  493. return 'http://' + url
  494. elif url[:8].lower() == 'https://':
  495. # Convert secure URLs to regular HTTP.
  496. return 'http://' + url[8:]
  497. else:
  498. # No changes required.
  499. return url
  500. def signature(self):
  501. """Return the server signature."""
  502. return 'WaybackProxy on {0}'.format(socket.gethostname())
  503. def wayback_to_datetime(self, date):
  504. """Convert a Wayback format date string to a datetime.datetime object."""
  505. try:
  506. return datetime.datetime.strptime(str(date)[:14], '%Y%m%d%H%M%S')
  507. except:
  508. return datetime.datetime.strptime(str(date)[:8], '%Y%m%d')
  509. print_lock = threading.Lock()
  510. def _print(*args, **kwargs):
  511. """Logging function."""
  512. if SILENT:
  513. return
  514. with print_lock:
  515. print(*args, **kwargs, flush=True)
  516. def main():
  517. """Starts the server."""
  518. server = ThreadingTCPServer(('', LISTEN_PORT), Handler)
  519. _print('[-] Now listening on port', LISTEN_PORT)
  520. _print('[-] Date set to', DATE)
  521. try:
  522. server.serve_forever()
  523. except KeyboardInterrupt: # Ctrl+C to stop
  524. pass
  525. if __name__ == '__main__':
  526. main()