Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 15 additions & 2 deletions handler.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,26 @@
import tornado.ioloop, tornado.web, tornado.autoreload
from tornado.escape import json_encode, json_decode

import safeurl, types, sys, re, mimetypes, glob, jsbeautifier, urlparse, pycurl
import safeurl, types, sys, re, mimetypes, glob, jsbeautifier, pycurl

# Python 2.7/3 urlparse
try:
# Python 2.7
from urlparse import urlparse
# from urllib import quote
except:
# Python 3
from urllib.parse import urlparse
# from urllib.parse import quote


import calendar, time, datetime

from netaddr import *
from collections import defaultdict
from bs4 import BeautifulSoup
from cgi import escape
# from cgi import escape
from html import escape

#------------------------------------------------------------
# Base / Status Code Handlers
Expand Down
12 changes: 7 additions & 5 deletions safeurl.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,9 @@
import re
import netaddr
import pycurl
import StringIO

# import StringIO
# This is python3
import io
# Python 2.7/3 urlparse
try:
# Python 2.7
Expand Down Expand Up @@ -673,7 +674,6 @@ def execute(self, url):
"""
# Backup the existing URL
originalUrl = url

# Execute, catch redirects and validate the URL
redirected = False
redirectCount = 0
Expand Down Expand Up @@ -707,7 +707,8 @@ def execute(self, url):
self._handle.setopt(pycurl.URL, url["cleanUrl"])

# Execute the cURL request
response = StringIO.StringIO()
# response = io.StringIO.StringIO()
response = io.BytesIO()
self._handle.setopt(pycurl.WRITEFUNCTION, response.write)
self._handle.perform()

Expand All @@ -729,4 +730,5 @@ def execute(self, url):
if not redirected:
break

return response.getvalue()
# return response.getvalue() # Python 2
return response.getvalue().decode('utf-8')
2 changes: 1 addition & 1 deletion setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,6 @@
long_description=open('README.md').read(),
author='Ben Sadeghipour',
url='https://github.com/nahamsec/JSParser',
install_requires=['safeurl', 'tornado', 'jsbeautifier',
install_requires=['safeurl', 'tornado<=5.1', 'jsbeautifier',
'netaddr', 'pycurl', 'BeautifulSoup4'],
)