waybackproxy.py 24 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577
  1. #!/usr/bin/env python3
  2. import base64, datetime, json, lrudict, re, socket, socketserver, sys, threading, traceback, urllib.request, urllib.error, urllib.parse
  3. from config import *
  4. class ThreadingTCPServer(socketserver.ThreadingMixIn, socketserver.TCPServer):
  5. """TCPServer with ThreadingMixIn added."""
  6. pass
  7. class SharedState:
  8. """Class for storing shared state across instances of Handler."""
  9. def __init__(self):
  10. # Create internal LRU dictionary for preserving URLs on redirect.
  11. self.date_cache = lrudict.LRUDict(maxduration=86400, maxsize=1024)
  12. # Create internal LRU dictionary for date availability.
  13. self.availability_cache = lrudict.LRUDict(maxduration=86400, maxsize=1024) if WAYBACK_API else None
  14. shared_state = SharedState()
  15. class Handler(socketserver.BaseRequestHandler):
  16. """Main request handler."""
  17. def setup(self, *args, **kwargs):
  18. """Set up this instance of Handler."""
  19. super().setup(*args, **kwargs)
  20. # Store a local pointer to SharedState.
  21. self.shared_state = shared_state
  22. def handle(self):
  23. """Handle a request."""
  24. # readline is pretty convenient
  25. f = self.request.makefile()
  26. # read request line
  27. reqline = line = f.readline()
  28. split = line.rstrip().split(' ')
  29. http_version = len(split) > 2 and split[2] or 'HTTP/0.9'
  30. if len(split) < 2 or split[0] != 'GET':
  31. # only GET is implemented
  32. return self.send_error_page(http_version, 501, 'Not Implemented')
  33. # read out the headers
  34. request_host = None
  35. pac_host = '" + location.host + ":' + str(LISTEN_PORT) # may not actually work
  36. effective_date = DATE
  37. auth = None
  38. while line.strip() != '':
  39. line = f.readline()
  40. ll = line.lower()
  41. if ll[:6] == 'host: ':
  42. pac_host = request_host = line[6:].rstrip()
  43. if ':' not in pac_host: # explicitly specify port if running on port 80
  44. pac_host += ':80'
  45. elif ll[:21] == 'x-waybackproxy-date: ':
  46. # API for a personal project of mine
  47. effective_date = line[21:].rstrip()
  48. elif ll[:21] == 'authorization: basic ':
  49. # asset date code passed as username:password
  50. auth = base64.b64decode(ll[21:])
  51. # parse the URL
  52. pac_file_paths = ('/proxy.pac', '/wpad.dat', '/wpad.da')
  53. if split[1][0] == '/' and split[1] not in pac_file_paths:
  54. # just a path (not corresponding to a PAC file) => transparent proxy
  55. # Host header and therefore HTTP/1.1 are required
  56. if not request_host:
  57. return self.send_error_page(http_version, 400, 'Host header missing')
  58. archived_url = 'http://' + request_host + split[1]
  59. else:
  60. # full URL => explicit proxy
  61. archived_url = split[1]
  62. request_url = archived_url
  63. parsed = urllib.parse.urlparse(request_url)
  64. # make a path
  65. path = parsed.path
  66. if parsed.query:
  67. path += '?' + parsed.query
  68. elif path == '':
  69. path == '/'
  70. # get the hostname for later
  71. host = parsed.netloc.split(':')
  72. hostname = host[0]
  73. # get cached date for redirects, if available
  74. original_date = effective_date
  75. effective_date = self.shared_state.date_cache.get(str(effective_date) + '\x00' + str(archived_url), effective_date)
  76. # get date from username:password, if available
  77. if auth:
  78. effective_date = auth.replace(':', '')
  79. # Effectively handle the request.
  80. try:
  81. if path in pac_file_paths:
  82. # PAC file to bypass QUICK_IMAGES requests if WAYBACK_API is not enabled.
  83. pac = http_version + ''' 200 OK\r\n'''
  84. pac += '''Content-Type: application/x-ns-proxy-autoconfig\r\n'''
  85. pac += '''\r\n'''
  86. pac += '''function FindProxyForURL(url, host)\r\n'''
  87. pac += '''{\r\n'''
  88. if self.shared_state.availability_cache == None:
  89. pac += ''' if (shExpMatch(url, "http://web.archive.org/web/*") && !shExpMatch(url, "http://web.archive.org/web/??????????????if_/*"))\r\n'''
  90. pac += ''' {\r\n'''
  91. pac += ''' return "DIRECT";\r\n'''
  92. pac += ''' }\r\n'''
  93. pac += ''' return "PROXY ''' + pac_host + '''";\r\n'''
  94. pac += '''}\r\n'''
  95. self.request.sendall(pac.encode('ascii', 'ignore'))
  96. return
  97. elif hostname == 'web.archive.org':
  98. if path[:5] != '/web/':
  99. # Launch settings if enabled.
  100. if SETTINGS_PAGE:
  101. return self.handle_settings(parsed.query)
  102. else:
  103. return self.send_error_page(http_version, 404, 'Not Found')
  104. else:
  105. # Pass requests through to web.archive.org. Required for QUICK_IMAGES.
  106. split = request_url.split('/')
  107. effective_date = split[4]
  108. archived_url = '/'.join(split[5:])
  109. _print('[>] [QI]', archived_url)
  110. elif GEOCITIES_FIX and hostname == 'www.geocities.com':
  111. # Apply GEOCITIES_FIX and pass it through.
  112. _print('[>]', archived_url)
  113. split = archived_url.split('/')
  114. hostname = split[2] = 'www.oocities.org'
  115. request_url = '/'.join(split)
  116. else:
  117. # Get from the Wayback Machine.
  118. _print('[>]', archived_url)
  119. request_url = 'http://web.archive.org/web/{0}/{1}'.format(effective_date, archived_url)
  120. # Check Wayback Machine Availability API where applicable, to avoid archived 404 pages and other site errors.
  121. if self.shared_state.availability_cache != None:
  122. # Are we requesting from the Wayback Machine?
  123. split = request_url.split('/')
  124. # If so, get the closest available date from the API.
  125. if split[2] == 'web.archive.org':
  126. # Remove extraneous :80 from URL.
  127. if ':' in split[5]:
  128. if split[7][-3:] == ':80':
  129. split[7] = split[7][:-3]
  130. elif split[5][-3:] == ':80':
  131. split[5] = split[5][:-3]
  132. # Check availability LRU cache.
  133. availability_url = '/'.join(split[5:])
  134. new_url = self.shared_state.availability_cache.get(availability_url, None)
  135. if new_url:
  136. # In cache => replace URL immediately.
  137. request_url = new_url
  138. else:
  139. # Not in cache => contact API.
  140. try:
  141. availability = json.loads(urllib.request.urlopen('https://archive.org/wayback/available?url=' + urllib.parse.quote_plus(availability_url) + '&timestamp=' + effective_date[:14], timeout=10).read())
  142. closest = availability.get('archived_snapshots', {}).get('closest', {})
  143. new_date = closest.get('timestamp', None)
  144. except:
  145. _print('[!] Failed to fetch Wayback availability data')
  146. new_date = None
  147. if new_date and new_date != effective_date[:14]:
  148. # Returned date is different.
  149. new_url = closest['url']
  150. # Add asset tag if one is present in the original URL.
  151. if len(effective_date) > 14:
  152. split = new_url.split('/')
  153. split[4] += effective_date[14:]
  154. new_url = '/'.join(split)
  155. # Replace URL and add it to the availability cache.
  156. request_url = self.shared_state.availability_cache[availability_url] = new_url
  157. # Start fetching the URL.
  158. conn = urllib.request.urlopen(request_url)
  159. except urllib.error.HTTPError as e:
  160. # An HTTP error has occurred.
  161. if e.code in (403, 404, 412): # not found or tolerance exceeded
  162. # Heuristically determine the static URL for some redirect scripts.
  163. match = re.search('''[^/]/((?:https?(?:%3A|:)(?:%2F|/)|www[0-9]*\\.[^/%]+)(?:%2F|/).+)''', archived_url, re.I) # URL in path
  164. if not match:
  165. match = re.search('''[\\?&][^=]+=((?:https?(?:%3A|:)(?:%2F|/)|www[0-9]*\\.[^/%]+)?(?:%2F|/)[^&]+)''', archived_url, re.I) # URL in query string
  166. if match: # found URL
  167. # Decode and sanitize the URL.
  168. new_url = self.sanitize_redirect(urllib.parse.unquote_plus(match.group(1)))
  169. # Redirect client to the URL.
  170. _print('[r] [g]', new_url)
  171. return self.send_redirect_page(http_version, new_url)
  172. elif e.code in (301, 302): # urllib-generated error about an infinite redirect loop
  173. _print('[!] Infinite redirect loop')
  174. return self.send_error_page(http_version, 508, 'Infinite Redirect Loop')
  175. if e.code != 412: # tolerance exceeded has its own error message above
  176. _print('[!]', e.code, e.reason)
  177. # If the memento Link header is present, this is a website error
  178. # instead of a Wayback error. Pass it along if that's the case.
  179. if 'Link' in e.headers:
  180. conn = e
  181. else:
  182. return self.send_error_page(http_version, e.code, e.reason)
  183. except socket.timeout as e:
  184. # A timeout has occurred.
  185. _print('[!] Fetch timeout')
  186. return self.send_error_page(http_version, 504, 'Gateway Timeout')
  187. except:
  188. # Some other fetch exception has occurred.
  189. _print('[!] Fetch exception:')
  190. traceback.print_exc()
  191. return self.send_error_page(http_version, 502, 'Bad Gateway')
  192. # Get content type.
  193. content_type = conn.info().get('Content-Type')
  194. if content_type == None:
  195. content_type = 'text/html'
  196. elif not CONTENT_TYPE_ENCODING:
  197. idx = content_type.find(';')
  198. if idx > -1:
  199. content_type = content_type[:idx]
  200. # Set the archive mode.
  201. if GEOCITIES_FIX and hostname in ('www.oocities.org', 'www.oocities.com'):
  202. mode = 1 # oocities
  203. else:
  204. mode = 0 # Wayback Machine
  205. # Check content type to determine if this is HTML we need to patch.
  206. # Wayback will add its HTML to anything it thinks is HTML.
  207. guessed_content_type = conn.info().get('X-Archive-Guessed-Content-Type')
  208. if not guessed_content_type:
  209. guessed_content_type = content_type
  210. if 'text/html' in guessed_content_type:
  211. # Some dynamically-generated links may end up pointing to
  212. # web.archive.org. Correct that by redirecting the Wayback
  213. # portion of the URL away if it ends up being HTML consumed
  214. # through the QUICK_IMAGES interface.
  215. if hostname == 'web.archive.org':
  216. conn.close()
  217. archived_url = '/'.join(request_url.split('/')[5:])
  218. _print('[r] [QI]', archived_url)
  219. return self.send_redirect_page(http_version, archived_url, 301)
  220. # Check if the date is within tolerance.
  221. if DATE_TOLERANCE is not None:
  222. match = re.search('''//web\\.archive\\.org/web/([0-9]+)''', conn.geturl())
  223. if match:
  224. requested_date = match.group(1)
  225. if self.wayback_to_datetime(requested_date) > self.wayback_to_datetime(original_date) + datetime.timedelta(int(DATE_TOLERANCE)):
  226. _print('[!]', requested_date, 'is outside the configured tolerance of', DATE_TOLERANCE, 'days')
  227. conn.close()
  228. return self.send_error_page(http_version, 412, 'Snapshot ' + requested_date + ' not available')
  229. # Consume all data.
  230. data = conn.read()
  231. # Patch the page.
  232. if mode == 0: # Wayback Machine
  233. # Check if this is a Wayback Machine page.
  234. if b'<title>Wayback Machine</title>' in data:
  235. # Check if this is an exclusion (robots.txt?) error page.
  236. if b'<p>This URL has been excluded from the Wayback Machine.</p>' in data:
  237. return self.send_error_page(http_version, 403, 'URL excluded')
  238. # Check if this is a media playback iframe page.
  239. # Some websites (especially ones that use frames)
  240. # inexplicably render inside a media playback iframe.
  241. # In that case, a simple redirect would result in a
  242. # redirect loop, so fetch and render the URL instead.
  243. match = re.search(b'''<iframe id="playback" src="((?:(?:https?:)?//web.archive.org)?/web/[^"]+)"''', data)
  244. if match:
  245. # Extract the content URL.
  246. request_url = match.group(1).decode('ascii', 'ignore')
  247. archived_url = '/'.join(request_url.split('/')[5:])
  248. # Start fetching the URL.
  249. _print('[f]', archived_url)
  250. try:
  251. conn = urllib.request.urlopen(request_url)
  252. except urllib.error.HTTPError as e:
  253. _print('[!]', e.code, e.reason)
  254. # If the memento Link header is present, this is a website error
  255. # instead of a Wayback error. Pass it along if that's the case.
  256. if 'Link' in e.headers:
  257. conn = e
  258. else:
  259. return self.send_error_page(http_version, e.code, e.reason)
  260. # Identify content type so we don't modify non-HTML content.
  261. content_type = conn.info().get('Content-Type')
  262. if not CONTENT_TYPE_ENCODING:
  263. idx = content_type.find(';')
  264. if idx > -1:
  265. content_type = content_type[:idx]
  266. if 'text/html' in content_type:
  267. # Consume all data and proceed with patching the page.
  268. data = conn.read()
  269. else:
  270. # Pass non-HTML data through.
  271. return self.send_passthrough(conn, http_version, content_type, request_url)
  272. # Check if this is a Wayback Machine redirect page.
  273. if b'<title></title>' in data and b'<span class="label style-scope media-button"><!---->Wayback Machine<!----></span>' in data:
  274. match = re.search(b'''<p class="impatient"><a href="(?:(?:https?:)?//web\\.archive\\.org)?/web/([^/]+)/([^"]+)">Impatient\\?</a></p>''', data)
  275. if match:
  276. # Sanitize the URL.
  277. archived_url = self.sanitize_redirect(match.group(2).decode('ascii', 'ignore'))
  278. # Add URL to the date LRU cache.
  279. self.shared_state.date_cache[str(effective_date) + '\x00' + archived_url] = match.group(1).decode('ascii', 'ignore')
  280. # Get the original HTTP redirect code.
  281. match = re.search(b'''<p class="code shift red">Got an HTTP ([0-9]+)''', data)
  282. try:
  283. redirect_code = int(match.group(1))
  284. except:
  285. redirect_code = 302
  286. # Redirect client to the URL.
  287. _print('[r]', archived_url)
  288. return self.send_redirect_page(http_version, archived_url, redirect_code)
  289. # Remove pre-toolbar scripts and CSS.
  290. data = re.sub(b'''<script src="//archive\\.org/.*<!-- End Wayback Rewrite JS Include -->\\r?\\n''', b'', data, flags=re.S)
  291. # Remove toolbar.
  292. data = re.sub(b'''<!-- BEGIN WAYBACK TOOLBAR INSERT -->.*<!-- END WAYBACK TOOLBAR INSERT -->''', b'', data, flags=re.S)
  293. # Remove comments on footer.
  294. data = re.sub(b'''<!--\\r?\\n FILE ARCHIVED .*$''', b'', data, flags=re.S)
  295. # Fix base tag.
  296. data = re.sub(b'''(<base\\s+[^>]*href=["']?)(?:(?:https?:)?//web.archive.org)?/web/[^/]+/(?:[^:/]+://)?''', b'\\1http://', data, flags=re.I + re.S)
  297. # Remove extraneous :80 from links.
  298. data = re.sub(b'((?:(?:https?:)?//web.archive.org)?/web/)([^/]+)/([^/:]+)://([^/:]+):80/', b'\\1\\2/\\3://\\4/', data)
  299. # Fix links.
  300. if QUICK_IMAGES:
  301. # QUICK_IMAGES works by intercepting asset URLs (those
  302. # with a date code ending in im_, js_...) and letting the
  303. # proxy pass them through. This may reduce load time
  304. # because Wayback doesn't have to hunt down the closest
  305. # copy of that asset to DATE, as those URLs have specific
  306. # date codes. This taints the HTML with web.archive.org
  307. # URLs. QUICK_IMAGES=2 uses the original URLs with an added
  308. # username:password, which taints less but is not supported
  309. # by all browsers - IE notably kills the whole page if it
  310. # sees an iframe pointing to an invalid URL.
  311. data = re.sub(b'(?:(?:https?:)?//web.archive.org)?/web/([0-9]+)([a-z]+_)/([^:/]+://)',
  312. QUICK_IMAGES == 2 and b'\\3\\1:\\2@' or b'http://web.archive.org/web/\\1\\2/\\3', data)
  313. def strip_https(match): # convert secure non-asset URLs to regular HTTP
  314. first_component = match.group(1)
  315. return first_component == b'https:' and b'http:' or first_component
  316. data = re.sub(b'(?:(?:https?:)?//web.archive.org)?/web/[^/]+/([^/]+)', strip_https, data)
  317. else:
  318. # Remove asset URLs while simultaneously adding them to the date LRU cache
  319. # with their respective date and converting secure URLs to regular HTTP.
  320. def add_to_date_cache(match):
  321. orig_url = match.group(2)
  322. if orig_url[:8] == b'https://':
  323. orig_url = b'http://' + orig_url[8:]
  324. self.shared_state.date_cache[str(effective_date) + '\x00' + orig_url.decode('ascii', 'ignore')] = match.group(1).decode('ascii', 'ignore')
  325. return orig_url
  326. data = re.sub(b'''(?:(?:https?:)?//web.archive.org)?/web/([^/]+)/([^"\\'#<>]+)''', add_to_date_cache, data)
  327. elif mode == 1: # oocities
  328. # Remove viewport/cache-control/max-width code from the header.
  329. data = re.sub(b'''^.*?\n\n''', b'', data, flags=re.S)
  330. # Remove archive notice and tracking code from the footer.
  331. data = re.sub(b'''<style> \n.zoomout { -webkit-transition: .*$''', b'', data, flags=re.S)
  332. # Remove clearly labeled snippets from Geocities.
  333. data = re.sub(b'''^.*<\\!-- text above generated by server\\. PLEASE REMOVE -->''', b'', data, flags=re.S)
  334. data = re.sub(b'''<\\!-- following code added by server\\. PLEASE REMOVE -->.*<\!-- preceding code added by server\. PLEASE REMOVE -->''', b'', data, flags=re.S)
  335. data = re.sub(b'''<\\!-- text below generated by server\\. PLEASE REMOVE -->.*$''', b'', data, flags=re.S)
  336. # Fix links.
  337. data = re.sub(b'''//([^\\.]*\\.)?oocities\\.com/''', b'//\\1geocities.com/', data, flags=re.S)
  338. # Send patched page.
  339. self.send_response_headers(conn, http_version, content_type, request_url)
  340. self.request.sendall(data)
  341. self.request.close()
  342. else:
  343. # Pass non-HTML data through.
  344. self.send_passthrough(conn, http_version, content_type, request_url)
  345. def send_passthrough(self, conn, http_version, content_type, request_url):
  346. """Pass data through to the client unmodified (save for our headers)."""
  347. self.send_response_headers(conn, http_version, content_type, request_url)
  348. while True:
  349. data = conn.read(1024)
  350. if not data:
  351. break
  352. self.request.sendall(data)
  353. self.request.close()
  354. def send_response_headers(self, conn, http_version, content_type, request_url):
  355. """Generate and send the response headers."""
  356. response = http_version
  357. # Pass the error code if there is one.
  358. if isinstance(conn, urllib.error.HTTPError):
  359. response += '{0} {1}'.format(conn.code, conn.reason.replace('\n', ' '))
  360. else:
  361. response += '200 OK'
  362. # Add content type, and the ETag for caching.
  363. response += '\r\nContent-Type: ' + content_type + '\r\nETag: "' + request_url.replace('"', '') + '"\r\n'
  364. # Add X-Archive-Orig-* headers.
  365. headers = conn.info()
  366. for header in headers:
  367. if header.find('X-Archive-Orig-') == 0:
  368. orig_header = header[15:]
  369. # Blacklist certain headers which may affect client behavior.
  370. if orig_header.lower() not in ('connection', 'location', 'content-type', 'content-length', 'etag', 'authorization', 'set-cookie'):
  371. response += orig_header + ': ' + headers[header] + '\r\n'
  372. # Finish and send the request.
  373. response += '\r\n'
  374. self.request.sendall(response.encode('ascii', 'ignore'))
  375. def send_error_page(self, http_version, code, reason):
  376. """Generate an error page."""
  377. # make error page
  378. errorpage = '<html><head><title>{0} {1}</title>'.format(code, reason)
  379. # IE's same-origin policy throws "Access is denied." inside frames
  380. # loaded from a different origin. Use that to our advantage, even
  381. # though regular frames are also affected. IE also doesn't recognize
  382. # language="javascript1.4", so use 1.3 while blocking IE4 by detecting
  383. # the lack of screenLeft as IE4 is quite noisy with script errors.
  384. errorpage += '<script language="javascript1.3">if (window.screenLeft != null) { eval(\'try { var frameElement = window.frameElement; } catch (e) { document.location.href = "about:blank"; }\'); }</script>'
  385. errorpage += '<script language="javascript">if (window.self != window.top && !(window.frameElement && window.frameElement.tagName == "FRAME")) { document.location.href = "about:blank"; }</script>'
  386. errorpage += '</head><body><h1>{0}</h1><p>'.format(reason)
  387. # add code information
  388. if code in (404, 508): # page not archived or redirect loop
  389. errorpage += 'This page may not be archived by the Wayback Machine.'
  390. elif code == 403: # not crawled due to exclusion
  391. errorpage += 'This page was not archived due to a Wayback Machine exclusion.'
  392. elif code == 501: # method not implemented
  393. errorpage += 'WaybackProxy only implements the GET method.'
  394. elif code == 502: # exception
  395. errorpage += 'This page could not be fetched due to an unknown error.'
  396. elif code == 504: # timeout
  397. errorpage += 'This page could not be fetched due to a Wayback Machine server timeout.'
  398. elif code == 412: # outside of tolerance
  399. errorpage += 'The earliest snapshot for this page is outside of the configured tolerance interval.'
  400. elif code == 400 and reason == 'Host header missing': # no host header in transparent mode
  401. errorpage += 'WaybackProxy\'s transparent mode requires an HTTP/1.1 compliant client.'
  402. else: # another error
  403. errorpage += 'Unknown error. The Wayback Machine may be experiencing technical difficulties.'
  404. errorpage += '</p><hr><i>'
  405. errorpage += self.signature()
  406. errorpage += '</i></body></html>'
  407. # add padding for IE
  408. if len(errorpage) <= 512:
  409. padding = '\n<!-- This comment pads the HTML so Internet Explorer displays this error page instead of its own. '
  410. remainder = 510 - len(errorpage) - len(padding)
  411. if remainder > 0:
  412. padding += ' ' * remainder
  413. padding += '-->'
  414. errorpage += padding
  415. # send error page and stop
  416. self.request.sendall('{0} {1} {2}\r\nContent-Type: text/html\r\nContent-Length: {3}\r\n\r\n{4}'.format(http_version, code, reason, len(errorpage), errorpage).encode('utf8', 'ignore'))
  417. self.request.close()
  418. def send_redirect_page(self, http_version, target, code=302):
  419. """Generate a redirect page."""
  420. # make redirect page
  421. redirectpage = '<html><head><title>Redirect</title><meta http-equiv="refresh" content="0;url='
  422. redirectpage += target
  423. redirectpage += '"></head><body><p>If you are not redirected, <a href="'
  424. redirectpage += target
  425. redirectpage += '">click here</a>.</p></body></html>'
  426. # send redirect page and stop
  427. self.request.sendall('{0} {1} Found\r\nLocation: {2}\r\nContent-Type: text/html\r\nContent-Length: {3}\r\n\r\n{4}'.format(http_version, code, target, len(redirectpage), redirectpage).encode('utf8', 'ignore'))
  428. self.request.close()
  429. def handle_settings(self, query):
  430. """Generate the settings page."""
  431. global DATE, DATE_TOLERANCE, GEOCITIES_FIX, QUICK_IMAGES, WAYBACK_API, CONTENT_TYPE_ENCODING, SILENT, SETTINGS_PAGE
  432. if query != '': # handle any parameters that may have been sent
  433. parsed = urllib.parse.parse_qs(query)
  434. if 'date' in parsed and 'dateTolerance' in parsed:
  435. if DATE != parsed['date'][0]:
  436. DATE = parsed['date'][0]
  437. self.shared_state.date_cache.clear()
  438. self.shared_state.availability_cache.clear()
  439. if DATE_TOLERANCE != parsed['dateTolerance'][0]:
  440. DATE_TOLERANCE = parsed['dateTolerance'][0]
  441. GEOCITIES_FIX = 'gcFix' in parsed
  442. QUICK_IMAGES = 'quickImages' in parsed
  443. CONTENT_TYPE_ENCODING = 'ctEncoding' in parsed
  444. # send the page and stop
  445. settingspage = 'HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n\r\n'
  446. settingspage += '<html><head><title>WaybackProxy Settings</title></head><body><p><b>'
  447. settingspage += self.signature()
  448. settingspage += '</b></p><form method="get" action="/">'
  449. settingspage += '<p>Date to get pages from: <input type="text" name="date" size="8" value="'
  450. settingspage += str(DATE)
  451. settingspage += '"><p>Date tolerance: <input type="text" name="dateTolerance" size="8" value="'
  452. settingspage += str(DATE_TOLERANCE)
  453. settingspage += '"> days<br><input type="checkbox" name="gcFix"'
  454. if GEOCITIES_FIX:
  455. settingspage += ' checked'
  456. settingspage += '> Geocities Fix<br><input type="checkbox" name="quickImages"'
  457. if QUICK_IMAGES:
  458. settingspage += ' checked'
  459. settingspage += '> Quick images<br><input type="checkbox" name="ctEncoding"'
  460. if CONTENT_TYPE_ENCODING:
  461. settingspage += ' checked'
  462. settingspage += '> Encoding in Content-Type</p><p><input type="submit" value="Save"></p></form></body></html>'
  463. self.request.send(settingspage.encode('utf8', 'ignore'))
  464. self.request.close()
  465. def sanitize_redirect(self, url):
  466. """Sanitize an URL for client-side redirection."""
  467. if url[0] != '/' and '://' not in url:
  468. # Add protocol if the URL is absolute but missing a protocol.
  469. return 'http://' + url
  470. elif url[:8].lower() == 'https://':
  471. # Convert secure URLs to regular HTTP.
  472. return 'http://' + url[8:]
  473. else:
  474. # No changes required.
  475. return url
  476. def signature(self):
  477. """Return the server signature."""
  478. return 'WaybackProxy on {0}'.format(socket.gethostname())
  479. def wayback_to_datetime(self, date):
  480. """Convert a Wayback format date string to a datetime.datetime object."""
  481. try:
  482. return datetime.datetime.strptime(str(date)[:14], '%Y%m%d%H%M%S')
  483. except:
  484. return datetime.datetime.strptime(str(date)[:8], '%Y%m%d')
  485. print_lock = threading.Lock()
  486. def _print(*args, **kwargs):
  487. """Logging function."""
  488. if SILENT:
  489. return
  490. with print_lock:
  491. print(*args, **kwargs, flush=True)
  492. def main():
  493. """Starts the server."""
  494. server = ThreadingTCPServer(('', LISTEN_PORT), Handler)
  495. _print('[-] Now listening on port', LISTEN_PORT)
  496. try:
  497. server.serve_forever()
  498. except KeyboardInterrupt: # Ctrl+C to stop
  499. pass
  500. if __name__ == '__main__':
  501. main()