[pull] master from sqlmapproject:master

data/txt/sha256sums.txt

@@ -84,7 +84,7 @@ b0f434f64105bd61ab0f6867b3f681b97fa02b4fb809ac538db382d031f0e609  data/xml/paylo
 0648264166455010921df1ec431e4c973809f37ef12cbfea75f95029222eb689  data/xml/payloads/stacked_queries.xml
 997556b6170964a64474a2e053abe33cf2cf029fb1acec660d4651cc67a3c7e1  data/xml/payloads/time_blind.xml
 40a4878669f318568097719d07dc906a19b8520bc742be3583321fc1e8176089  data/xml/payloads/union_query.xml
-8b63fda09d5c5e43ad8e6db1db90e5b1017fbe02735f3858843fc52118e3a33a  data/xml/queries.xml
+9d7dcbc6c5e368c44db851865ff49c791c3dee1ee62d8c02af8f8b15f4551aed  data/xml/queries.xml
 0f5a9c84cb57809be8759f483c7d05f54847115e715521ac0ecf390c0aa68465  doc/AUTHORS
 ce20a4b452f24a97fde7ec9ed816feee12ac148e1fde5f1722772cc866b12740  doc/CHANGELOG.md
 233fb10dff24a2436eb24496db7fadb46659da6745a0d53c744db701188041ef  doc/THANKS.md
@@ -118,7 +118,7 @@ c4590a37dc1372be29b9ba8674b5e12bcda6ab62c5b2d18dab20bcb73a4ffbeb  doc/translatio
 8c4b528855c2391c91ec1643aeff87cae14246570fd95dac01b3326f505cd26e  extra/beep/beep.py
 509276140d23bfc079a6863e0291c4d0077dea6942658a992cbca7904a43fae9  extra/beep/beep.wav
 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3  extra/beep/__init__.py
-b8d919ad6c632a9f5b292ee6c0476e9b092a39c0727fe89d12102d1938217116  extra/cloak/cloak.py
+7f6394c9b3566bf93fc10020bc584aa8fac36dc11c3c523096eadc63ab243ec9  extra/cloak/cloak.py
 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3  extra/cloak/__init__.py
 6879b01859b2003fbab79c5188fce298264cd00300f9dcecbe1ffd980fe2e128  extra/cloak/README.txt
 4b6d44258599f306186a24e99d8648d94b04d85c1f2c2a442b15dc26d862b41e  extra/dbgtool/dbgtool.py
@@ -168,8 +168,8 @@ d69e84f1648cdb907f5d2dd454f03874a4613752b07867510145d51d84b3c56f  lib/controller
 bc655c5f09a4048e53d2fec5f65e9e45024c2ad9882b8824b0d338917fd6496b  lib/core/agent.py
 ca3e5ce56cb1cae0a8e815425ab6810068004bffe8861d1037c7c87c0ae02477  lib/core/bigarray.py
 c91b6b9429a50d28b88334e3f88557d40a01893a7e69c30186c2f6efd0ce9906  lib/core/common.py
-f30b4eccdb574731fa7e6ef48e71ea82d4bc99be70a2e27bff230943e9039313  lib/core/compat.py
-e37bfd314a46699b14e1c8a5ea851d546d3a36bea8e5f37466ef2921ff78fefd  lib/core/convert.py
+8f1272487e1adfcc8c755a2f56f0c6d21eac5e685a73a9a159482f9dc9142bc5  lib/core/compat.py
+742bce10b97034966021ec60c7ac294db4af4fe7893613d63172a02c29f009f8  lib/core/convert.py
 c03dc585f89642cfd81b087ac2723e3e1bb3bfa8c60e6f5fe58ef3b0113ebfe6  lib/core/data.py
 6acb645b1f285b21673c70824b03f6209acc5993b50e50da5ed2c713a30626f5  lib/core/datatype.py
 70fb2528e580b22564899595b0dff6b1bc257c6a99d2022ce3996a3d04e68e4e  lib/core/decorators.py
@@ -182,13 +182,13 @@ c03dc585f89642cfd81b087ac2723e3e1bb3bfa8c60e6f5fe58ef3b0113ebfe6  lib/core/data.
 914a13ee21fd610a6153a37cbe50830fcbd1324c7ebc1e7fc206d5e598b0f7ad  lib/core/log.py
 67ea32c993cbf23cdbd5170360c020ca33363b7c516ff3f8da4124ef7cb0254d  lib/core/optiondict.py
 3ff871fe8391952c3ec3bb528ba592a13926c80ca0b68fd322a317f69a651ef7  lib/core/option.py
-3371a9c79ad7d2eb578e705cb077098a9f63cabb5472e4e66c4dac094a438bcd  lib/core/patch.py
+2e66d74a4d9adb9ce30f48e22ab83b7fdccb54e7ea7b74a6104bda7d80a71a7a  lib/core/patch.py
 49c0fa7e3814dfda610d665ee02b12df299b28bc0b6773815b4395514ddf8dec  lib/core/profiling.py
 03db48f02c3d07a047ddb8fe33a757b6238867352d8ddda2a83e4fec09a98d04  lib/core/readlineng.py
 48797d6c34dd9bb8a53f7f3794c85f4288d82a9a1d6be7fcf317d388cb20d4b3  lib/core/replication.py
 0b8c38a01bb01f843d94a6c5f2075ee47520d0c4aa799cecea9c3e2c5a4a23a6  lib/core/revision.py
 888daba83fd4a34e9503fe21f01fef4cc730e5cde871b1d40e15d4cbc847d56c  lib/core/session.py
-1c130171ed2b53992204d421c7dc663cc122bf16ec73f544ba72c7b3e7487ff9  lib/core/settings.py
+40244898d0eb5e2634a6794d78fa29315e9e4b9c6f773133a29dd20259bc63a0  lib/core/settings.py
 cd5a66deee8963ba8e7e9af3dd36eb5e8127d4d68698811c29e789655f507f82  lib/core/shell.py
 bcb5d8090d5e3e0ef2a586ba09ba80eef0c6d51feb0f611ed25299fbb254f725  lib/core/subprocessng.py
 70ea3768f1b3062b22d20644df41c86238157ec80dd43da40545c620714273c6  lib/core/target.py
@@ -203,12 +203,12 @@ b9aacb840310173202f79c2ba125b0243003ee6b44c92eca50424f2bdfc83c02  lib/core/unesc
 02d82e4069bd98c52755417f8b8e306d79945672656ac24f1a45e7a6eff4b158  lib/parse/configfile.py
 c5b258be7485089fac9d9cd179960e774fbd85e62836dc67cce76cc028bb6aeb  lib/parse/handler.py
 5c9a9caee948843d5537745640cc7b98d70a0412cc0949f59d4ebe8b2907c06c  lib/parse/headers.py
-1ad9054cd8476a520d4e2c141085ae45d94519df5c66f25fac41fe7d552ab952  lib/parse/html.py
+ea9b195e5f5030b96d1993c106c1e13fb5c7faaf6bdc5daacfd06ec984e7f323  lib/parse/html.py
 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3  lib/parse/__init__.py
 d2e771cdacef25ee3fdc0e0355b92e7cd1b68f5edc2756ffc19f75d183ba2c73  lib/parse/payloads.py
-455ab0ec63e55cd56ce4a884b85bdc089223155008cab0f3696da5a33118f95b  lib/parse/sitemap.py
+c2f34e27578742e729c2fa9c1d4f0a0d8f8f7f4cf0fc14c62ec817a260c71dec  lib/parse/sitemap.py
 1be3da334411657461421b8a26a0f2ff28e1af1e28f1e963c6c92768f9b0847c  lib/request/basicauthhandler.py
-de8e087e041e3252e6dd60d171a0cfe349f1e11764274108e8e13ba5be992ef3  lib/request/basic.py
+369484a2999d29f49bf839a329d1686ed94f6ea27c695e027fe08c8da51f30a3  lib/request/basic.py
 bc61bc944b81a7670884f82231033a6ac703324b34b071c9834886a92e249d0e  lib/request/chunkedhandler.py
 09c2d8786fb5280f5f14a7b4345ecb2e7c2ca836ee06a6cf9b51770df923d94c  lib/request/comparison.py
 c4a0759ee29ce8a29648090660dc273494abef9bda52430c38e41675a9b6ac6a  lib/request/connect.py
@@ -241,23 +241,23 @@ f552b6140d4069be6a44792a08f295da8adabc1c4bb6a5e100f222f87144ca9d  lib/techniques
 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3  lib/techniques/union/__init__.py
 30cae858e2a5a75b40854399f65ad074e6bb808d56d5ee66b94d4002dc6e101b  lib/techniques/union/test.py
 a8a795f29ec6fd66482926f04b054ed492a033982c3b7837c5d2ea32368acec0  lib/techniques/union/use.py
-5832f1b9cce5e8fe71cc1e07a690fa30f2bc0caa07e734220372a846aae6b95f  lib/utils/api.py
+7c33894b640d93fc8062781525586791479c9984c3de04283826642e5c7c4374  lib/utils/api.py
 442555ab85277aff7c9e0cf465ea5b0d28395c326f68363449b2d3941f4b6de2  lib/utils/brute.py
 da5bcbcda3f667582adf5db8c1b5d511b469ac61b55d387cec66de35720ed718  lib/utils/crawler.py
 a94958be0ec3e9d28d8171813a6a90655a9ad7e6aa33c661e8d8ebbfcf208dbb  lib/utils/deps.py
 51cfab194cd5b6b24d62706fb79db86c852b9e593f4c55c15b35f175e70c9d75  lib/utils/getch.py
 853c3595e1d2efc54b8bfb6ab12c55d1efc1603be266978e3a7d96d553d91a52  lib/utils/gui.py
 972c5db9c9e30ac0f91c0f8d4df4531d0304e151dac99f1399c37c952ba9f935  lib/utils/har.py
-b74a311e1cd30ec62e54684f970c14bfd85ffde225b9ddbbb12b85f3c528f8c2  lib/utils/hashdb.py
+0cd3860c03e39bacd1d0fe4cf1a0c605de48ff82f70441319f21d47e38e7e3a9  lib/utils/hashdb.py
 71a66ff766a2921106770b26acff380de469222dc893816a7b970b384c927666  lib/utils/hash.py
 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3  lib/utils/__init__.py
 e7d31de0e268c129ee11c590eb618f73a85e1022c08b8ed1f77753043c949214  lib/utils/pivotdumptable.py
 c1dfc3bed0fed9b181f612d1d747955dd2b506dbe99bc9fd481495602371473a  lib/utils/progress.py
-27afe211030d06db28df85296bfbf698296c94440904c390cef0ff0c259dbbc5  lib/utils/purge.py
+2cd84db16edef8c9948e197a51d870cf1c338f4a89037b4d422de990f4a45237  lib/utils/purge.py
 f635872093a12cd63a72d77adf88e8f8cd4084a5cc64384f12966cd75a499bdf  lib/utils/safe2bin.py
 de4be7e291db0962cd59f9c04b3f7259f846e315df1fd9b323954f89fae0b2db  lib/utils/search.py
 8258d0f54ad94e6101934971af4e55d5540f217c40ddcc594e2fba837b856d35  lib/utils/sgmllib.py
-92361b3c14ca472f0f89c275814da021c4f0e2de6ffa1bffc691b4cdc38d59dc  lib/utils/sqlalchemy.py
+2760c4b82382e501f16bb98edec9531f46e5b286fbf004b346545b9b62f84824  lib/utils/sqlalchemy.py
 f0e5525a92fe971defc8f74c27942ff9138b1e8251f2e0d9a8bd59285b656084  lib/utils/timeout.py
 f821dc39a75ea48dccfa758788de15d38b9ca6a780a98f59935fb6610f75508c  lib/utils/tui.py
 e430db49aa768ff2cdba76932e30871c366054599c44d91580dde459ab9b6fef  lib/utils/versioncheck.py

data/xml/queries.xml

@@ -1713,6 +1713,7 @@
         <concatenate query="%s||%s"/>
         <case query="SELECT (IF(%s,1,0))"/>
         <inference query="UNICODE(SUBSTRING((%s),%d,1))>%d"/>
+        <substring query="SUBSTRING((%s),%d,%d)"/>
         <banner/>
         <current_user/>
         <current_db/>
@@ -1752,6 +1753,7 @@
         <concatenate query="%s||%s"/>
         <case query="SELECT (CASE WHEN (%s) THEN 1 ELSE 0 END)"/>
         <inference query="ASCII(SUBSTRING((%s),%d,1))>%d"/>
+        <substring query="SUBSTRING((%s),%d,%d)"/>
         <banner query="sys_stat('st_dbms_name')||' - '||sys_stat('st_dbms_ver')"/>
         <current_user query="USERNAME()"/>
         <current_db query="UPPER(USERNAME())"/>
@@ -1803,6 +1805,7 @@
         <concatenate query="%s||%s"/>
         <case query="SELECT (CASE WHEN (%s) THEN 1 ELSE 0 END)"/>
         <inference query="ASCII(SUBSTR((%s),%d,1))>%d"/>
+        <substring query="SUBSTR((%s),%d,%d)"/>
         <banner query="CURRENT_VERSION()"/>
         <current_user query="CURRENT_USER()"/>
         <current_db query="CURRENT_SCHEMA()"/>

extra/cloak/cloak.py

@@ -43,8 +43,6 @@ def decloak(inputFile=None, data=None):
         print(ex)
         print('ERROR: the provided input file \'%s\' does not contain valid cloaked content' % inputFile)
         sys.exit(1)
-    finally:
-        f.close()
 
     return data
 

lib/core/compat.py

@@ -280,6 +280,12 @@ def __hash__(self):
     xrange = xrange
     buffer = buffer
 
+try:
+    RecursionError = RecursionError
+except NameError:
+    # Note: patch for Python < 3.5 (RecursionError, a subclass of RuntimeError, was introduced in Python 3.5)
+    RecursionError = RuntimeError
+
 def LooseVersion(version):
     """
     >>> LooseVersion("1.0") == LooseVersion("1.0")

lib/core/convert.py

@@ -45,6 +45,8 @@ def base64pickle(value):
 
     >>> base64unpickle(base64pickle([1, 2, 3])) == [1, 2, 3]
     True
+    >>> isinstance(base64unpickle(base64pickle(BigArray([1, 2, 3]))), BigArray)
+    True
     """
 
     retVal = None

lib/core/patch.py

@@ -182,15 +182,17 @@ def reject(*args): raise ValueError("XML entities are forbidden")
     import pickle
     if not getattr(pickle, "_patched", False):
         class RestrictedUnpickler(pickle.Unpickler):
+            # Note: allowlist (not blacklist) - a module blacklist is bypassable (e.g. importlib/ctypes/operator), so only
+            # explicitly-safe builtin data types and sqlmap's own (and bundled) classes are permitted to be unpickled
             def find_class(self, module, name):
-                # blacklist for OS-level execution modules
-                if module in ("os", "subprocess", "sys", "posix", "nt", "pty", "commands", "shutil"):
+                # safe builtin data types only (blocks eval/exec/__import__/getattr/etc.)
+                if module in ("builtins", "__builtin__"):
+                    if name not in ("set", "frozenset", "dict", "list", "tuple", "int", "float", "bool", "str", "bytes", "bytearray", "object", "NoneType", "complex"):
+                        raise ValueError("unpickling of '%s.%s' is forbidden" % (module, name))
+                # everything else must be one of sqlmap's own (or bundled) classes (e.g. lib.core.datatype.AttribDict)
+                elif (module or "").split(".")[0] not in ("lib", "plugins", "thirdparty"):
                     raise ValueError("unpickling of module '%s' is forbidden" % module)
 
-                # partial whitelist for builtins to allow safe data types but block eval/exec/__import__
-                if module in ("builtins", "__builtin__") and name not in ("set", "frozenset", "dict", "list", "tuple", "int", "float", "bool", "str", "bytes", "bytearray", "object", "NoneType"):
-                    raise ValueError("unpickling of '%s.%s' is forbidden" % (module, name))
-
                 # Python 2/3 method resolution
                 if hasattr(pickle.Unpickler, "find_class"):
                     return pickle.Unpickler.find_class(self, module, name)

lib/core/settings.py

@@ -20,7 +20,7 @@
 from thirdparty import six
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.10.6.97"
+VERSION = "1.10.6.100"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)
@@ -681,6 +681,9 @@
 # Reference: https://web.archive.org/web/20150407141500/https://support.microsoft.com/en-us/kb/899149
 DUMP_FILE_BUFFER_SIZE = 1024
 
+# Block size used for the in-place secure-overwrite passes of '--purge' (bounds peak memory regardless of file size)
+PURGE_BLOCK_SIZE = 1024 * 1024
+
 # Parse response headers only first couple of times
 PARSE_HEADERS_LIMIT = 3
 

lib/parse/html.py

@@ -27,11 +27,11 @@ def __init__(self, page):
 
         self._dbms = None
         self._page = (page or "")
+        self._urldecodedPage = urldecode(self._page)
         try:
-            self._lower_page = self._page.lower()
+            self._lowerPage = self._urldecodedPage.lower()  # Note: keyword pre-filter must match the page that re.search() runs on (the URL-decoded one)
         except SystemError:  # https://bugs.python.org/issue18183
-            self._lower_page = None
-        self._urldecoded_page = urldecode(self._page)
+            self._lowerPage = None
 
         self.dbms = None
 
@@ -53,7 +53,7 @@ def startElement(self, name, attrs):
                 keywords = sorted(keywords, key=len)
                 kb.cache.regex[regexp] = keywords[-1].lower()
 
-            if ('|' in regexp or kb.cache.regex[regexp] in (self._lower_page or kb.cache.regex[regexp])) and re.search(regexp, self._urldecoded_page, re.I):
+            if ('|' in regexp or kb.cache.regex[regexp] in (self._lowerPage or kb.cache.regex[regexp])) and re.search(regexp, self._urldecodedPage, re.I):
                 self.dbms = self._dbms
                 self._markAsErrorPage()
                 kb.forkNote = kb.forkNote or attrs.get("fork")

lib/parse/sitemap.py

@@ -41,7 +41,7 @@ def parseSitemap(url, retVal=None, visited=None):
             raise SqlmapSyntaxException(errMsg)
 
         if content:
-            content = re.sub(r"", "", content, flags=re.DOTALL)
+            content = re.sub(r"<!--.*?-->", "", content, flags=re.DOTALL)  # Note: strip (possibly multi-line) XML comments so commented-out <loc> entries aren't harvested
 
             for match in re.finditer(r"<\w*?loc[^>]*>\s*([^<]+)", content, re.I):
                 if abortedFlag:

lib/request/basic.py

@@ -284,6 +284,8 @@ def decodePage(page, contentEncoding, contentType, percentDecode=True):
     '\\t'
     >>> getText(decodePage(b"J", None, "text/html; charset=utf-8"))
     'J'
+    >>> decodePage(b"™", None, "text/html; charset=utf-8") == u"\u2122"
+    True
     """
 
     if not page or (conf.nullConnection and len(page) < 2):
@@ -379,6 +381,16 @@ def _(match):
                     return retVal
                 page = re.sub(r"&#(\d+);", _, page)
 
+                # e.g. &#x2019;&#x2026;&#x2122; (hex numeric refs >= U+0100; smaller ones already handled at byte-level)
+                def _(match):
+                    retVal = match.group(0)
+                    try:
+                        retVal = _unichr(int(match.group(1), 16))
+                    except (ValueError, OverflowError):
+                        pass
+                    return retVal
+                page = re.sub(r"(?i)&#x([0-9a-f]+);", _, page)
+
             # e.g. &zeta;
             page = re.sub(r"&([^;]+);", lambda _: _unichr(HTML_ENTITIES[_.group(1)]) if HTML_ENTITIES.get(_.group(1), 0) > 255 else _.group(0), page)
         else:

lib/utils/api.py

@@ -327,10 +327,10 @@ def check_authentication():
     except:
         request.environ["PATH_INFO"] = "/error/401"
     else:
-        if creds.count(':') != 1:
+        if ':' not in creds:
             request.environ["PATH_INFO"] = "/error/401"
         else:
-            username, password = creds.split(':')
+            username, password = creds.split(':', 1)
             if username.strip() != (DataStore.username or "") or password.strip() != (DataStore.password or ""):
                 request.environ["PATH_INFO"] = "/error/401"
 

lib/utils/hashdb.py

@@ -16,6 +16,7 @@
 from lib.core.common import serializeObject
 from lib.core.common import singleTimeWarnMessage
 from lib.core.common import unserializeObject
+from lib.core.compat import RecursionError
 from lib.core.compat import xrange
 from lib.core.convert import getBytes
 from lib.core.convert import getUnicode

lib/utils/purge.py

@@ -13,11 +13,9 @@
 import string
 
 from lib.core.common import getSafeExString
-from lib.core.common import openFile
-from lib.core.compat import xrange
 from lib.core.convert import getUnicode
 from lib.core.data import logger
-from thirdparty.six import unichr as _unichr
+from lib.core.settings import PURGE_BLOCK_SIZE
 
 def purge(directory):
     """
@@ -46,12 +44,25 @@ def purge(directory):
         except:
             pass
 
-    logger.debug("writing random data to files")
+    logger.debug("overwriting file contents")
     for filepath in filepaths:
         try:
             filesize = os.path.getsize(filepath)
-            with openFile(filepath, "w+") as f:
-                f.write("".join(_unichr(random.randint(0, 255)) for _ in xrange(filesize)))
+            if filesize:
+                # Note: NIST SP 800-88 ("Clear") / DoD 5220.22-M style multi-pass in-place overwrite
+                # (zeros, ones, random) forcing each pass to disk; performed BEFORE the truncation below
+                # so the original bytes are actually overwritten and not just released to free blocks.
+                # Written in bounded blocks so peak memory stays O(PURGE_BLOCK_SIZE), not O(filesize)
+                with open(filepath, "r+b") as f:
+                    for getBlock in (lambda n: b"\x00" * n, lambda n: b"\xff" * n, lambda n: os.urandom(n)):
+                        f.seek(0)
+                        remaining = filesize
+                        while remaining > 0:
+                            count = min(PURGE_BLOCK_SIZE, remaining)
+                            f.write(getBlock(count))
+                            remaining -= count
+                        f.flush()
+                        os.fsync(f.fileno())
         except:
             pass
 

lib/utils/sqlalchemy.py

@@ -82,18 +82,22 @@ def connect(self):
                     engine = _sqlalchemy.create_engine(self.address, connect_args={})
 
                 self.connector = engine.connect()
-            except (TypeError, ValueError):
+            except (TypeError, ValueError) as ex:
                 if "_get_server_version_info" in traceback.format_exc():
                     try:
                         import pymssql
                         if int(pymssql.__version__[0]) < 2:
                             raise SqlmapConnectionException("SQLAlchemy connection issue (obsolete version of pymssql ('%s') is causing problems)" % pymssql.__version__)
                     except ImportError:
                         pass
+                    # Note: surface (as a proper SqlmapConnectionException) instead of silently continuing with self.connector left None
+                    raise SqlmapConnectionException("SQLAlchemy connection issue ('%s')" % getSafeExString(ex))
                 elif "invalid literal for int() with base 10: '0b" in traceback.format_exc():
                     raise SqlmapConnectionException("SQLAlchemy connection issue ('https://bitbucket.org/zzzeek/sqlalchemy/issues/3975')")
                 else:
-                    pass
+                    # Note: raise as SqlmapConnectionException (like the generic handler below) so the caller's native-connector
+                    # fallback engages and no raw TypeError/ValueError can reach sqlmap's top-level handler
+                    raise SqlmapConnectionException("SQLAlchemy connection issue ('%s')" % getSafeExString(ex))
             except SqlmapFilePathException:
                 raise
             except Exception as ex: