diff --git a/data/txt/sha256sums.txt b/data/txt/sha256sums.txt index 35b989ed02..a5091a6f11 100644 --- a/data/txt/sha256sums.txt +++ b/data/txt/sha256sums.txt @@ -161,13 +161,13 @@ df768bcb9838dc6c46dab9b4a877056cb4742bd6cfaaf438c4a3712c5cc0d264 extra/shutils/ 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3 extra/vulnserver/__init__.py 9e5e4d3d9acb767412259895a3ee75e1a5f42d0b9923f17605d771db384a6f60 extra/vulnserver/vulnserver.py b8411d1035bb49b073476404e61e1be7f4c61e205057730e2f7880beadcd5f60 lib/controller/action.py -03239569ebcdcb4c445bc778abb8f6fc7e26285a872d302cf3d366fb1c0c85b1 lib/controller/checks.py +6da812281a69c8b7a5181c2f76374dc695e4727b2936042651bacbeda4e6bcc9 lib/controller/checks.py c1881685bef8504ded32c51abed00ab51849008c84b74e8a66117e5f5041b3df lib/controller/controller.py d69e84f1648cdb907f5d2dd454f03874a4613752b07867510145d51d84b3c56f lib/controller/handler.py 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3 lib/controller/__init__.py -b2555d11529689f5d7d02bee0741d3228969e2bf29a2b9140bf1560ff60249e7 lib/core/agent.py +bc655c5f09a4048e53d2fec5f65e9e45024c2ad9882b8824b0d338917fd6496b lib/core/agent.py ca3e5ce56cb1cae0a8e815425ab6810068004bffe8861d1037c7c87c0ae02477 lib/core/bigarray.py -d3993ce0d1c73150d87405d1d4479bff189ecad179614b76c216dde65a0e06cd lib/core/common.py +c91b6b9429a50d28b88334e3f88557d40a01893a7e69c30186c2f6efd0ce9906 lib/core/common.py f30b4eccdb574731fa7e6ef48e71ea82d4bc99be70a2e27bff230943e9039313 lib/core/compat.py e37bfd314a46699b14e1c8a5ea851d546d3a36bea8e5f37466ef2921ff78fefd lib/core/convert.py c03dc585f89642cfd81b087ac2723e3e1bb3bfa8c60e6f5fe58ef3b0113ebfe6 lib/core/data.py @@ -181,14 +181,14 @@ c03dc585f89642cfd81b087ac2723e3e1bb3bfa8c60e6f5fe58ef3b0113ebfe6 lib/core/data. 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3 lib/core/__init__.py 914a13ee21fd610a6153a37cbe50830fcbd1324c7ebc1e7fc206d5e598b0f7ad lib/core/log.py 67ea32c993cbf23cdbd5170360c020ca33363b7c516ff3f8da4124ef7cb0254d lib/core/optiondict.py -83ec82a78c1665ae7516a3bbd239ffb1db8ac2ca20994125ff6023edf3d1e7c1 lib/core/option.py +3ff871fe8391952c3ec3bb528ba592a13926c80ca0b68fd322a317f69a651ef7 lib/core/option.py 3371a9c79ad7d2eb578e705cb077098a9f63cabb5472e4e66c4dac094a438bcd lib/core/patch.py 49c0fa7e3814dfda610d665ee02b12df299b28bc0b6773815b4395514ddf8dec lib/core/profiling.py 03db48f02c3d07a047ddb8fe33a757b6238867352d8ddda2a83e4fec09a98d04 lib/core/readlineng.py 48797d6c34dd9bb8a53f7f3794c85f4288d82a9a1d6be7fcf317d388cb20d4b3 lib/core/replication.py 0b8c38a01bb01f843d94a6c5f2075ee47520d0c4aa799cecea9c3e2c5a4a23a6 lib/core/revision.py 888daba83fd4a34e9503fe21f01fef4cc730e5cde871b1d40e15d4cbc847d56c lib/core/session.py -5d7a0d270747665518827e61ac39282434cf44d1b5a65a508c2ec8ed96d6b5fe lib/core/settings.py +dcd48ee4483e0ef394e4d4f7bb5566b107ea0e0d4b665ad2b0280ecb12799b47 lib/core/settings.py cd5a66deee8963ba8e7e9af3dd36eb5e8127d4d68698811c29e789655f507f82 lib/core/shell.py bcb5d8090d5e3e0ef2a586ba09ba80eef0c6d51feb0f611ed25299fbb254f725 lib/core/subprocessng.py 70ea3768f1b3062b22d20644df41c86238157ec80dd43da40545c620714273c6 lib/core/target.py @@ -230,7 +230,7 @@ f522436fbd14bdab090a1d305fcac0361800cb8e36c8cbcb47933298376a71e0 lib/takeover/r 0787f78e6bd9bb21d4267c95c4c99806711bb57c5518485c2e25f10fcf9c41fc lib/takeover/udf.py 23d73af417604dab460b74cdc230896153f018a6c00d144019491053640a172f lib/takeover/web.py 8cc1e226d4150fe8aa1a056e5d32d858ed6444d3d4e2af7fb4bc08f0bbe9d527 lib/takeover/xp_cmdshell.py -3609556c6c72010ce4cae5ffeeb74437a15a9dc218f77e079655f32e704fdeef lib/techniques/blind/inference.py +ea815192edb20b5f60e72a7eded9e2942c9e1dcb378b86f101ee69cf8de149f3 lib/techniques/blind/inference.py 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3 lib/techniques/blind/__init__.py 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3 lib/techniques/dns/__init__.py 3df9839fb92a81d46b6194d7adacb43f391efb78b071783c132e8d596ecbfaf1 lib/techniques/dns/test.py @@ -254,14 +254,14 @@ b74a311e1cd30ec62e54684f970c14bfd85ffde225b9ddbbb12b85f3c528f8c2 lib/utils/hash e7d31de0e268c129ee11c590eb618f73a85e1022c08b8ed1f77753043c949214 lib/utils/pivotdumptable.py c1dfc3bed0fed9b181f612d1d747955dd2b506dbe99bc9fd481495602371473a lib/utils/progress.py 27afe211030d06db28df85296bfbf698296c94440904c390cef0ff0c259dbbc5 lib/utils/purge.py -c853aa08ab24a00a78969408d60684da0ccb33a2a6693492e0acb7c480ffbcd1 lib/utils/safe2bin.py -2ee72e83500a1bf02fcd942564fca0053a0c46f736286f0c35dd6904e09f4734 lib/utils/search.py +f635872093a12cd63a72d77adf88e8f8cd4084a5cc64384f12966cd75a499bdf lib/utils/safe2bin.py +de4be7e291db0962cd59f9c04b3f7259f846e315df1fd9b323954f89fae0b2db lib/utils/search.py 8258d0f54ad94e6101934971af4e55d5540f217c40ddcc594e2fba837b856d35 lib/utils/sgmllib.py -b08373d647f337722983221d9051d8da253bf02e3f084aba8aee642ace8d02a6 lib/utils/sqlalchemy.py +92361b3c14ca472f0f89c275814da021c4f0e2de6ffa1bffc691b4cdc38d59dc lib/utils/sqlalchemy.py f0e5525a92fe971defc8f74c27942ff9138b1e8251f2e0d9a8bd59285b656084 lib/utils/timeout.py f821dc39a75ea48dccfa758788de15d38b9ca6a780a98f59935fb6610f75508c lib/utils/tui.py e430db49aa768ff2cdba76932e30871c366054599c44d91580dde459ab9b6fef lib/utils/versioncheck.py -b6cd3059c369bbcb162cfd797596849f9f95078c3b2e91fecee36d3ea1001fc2 lib/utils/xrange.py +1b439fc59fd202c21c74978ed9f36d1c309533226c77907eae159461525f9fef lib/utils/xrange.py b1bbb62f5b272a6247d442d5e4f644a5bca7138e70776539ec84a5a90433fd13 LICENSE 6b1828a80ae3472f1adb53a540dee0835eccac14f8cfc4bf73962c4e49a49557 plugins/dbms/access/connector.py c18939660aebb5ce323b4c78a46a2b119869ba8d0b44c853924118936ce5b0ac plugins/dbms/access/enumeration.py @@ -498,27 +498,27 @@ a9785a4c111d6fee2e6d26466ba5efb3b229c00520b26e8024b041553b53efba tamper/apostro cf26bc8006519bd25ce06d347f72770cd75b61575cf65e5812274e8ab9392eb4 tamper/apostrophenullencode.py 0b9ed12565bf000c9daa2317e915f2325ccabee1fa5ed5552c0787733fbccffe tamper/appendnullbyte.py 11ad15d66c43f32f5d0a39052e5f623a4752ad4fb275d642f2e4cd841ff82b41 tamper/base64encode.py -cb833979eccf26a5e176f7c8ca40a24bf9904cb2902a1b9df436aefb6a24447e tamper/between.py +1b55b7c59c623411c8cf328fff9e7de96a2dfc48ef4e5455325bfd41aebbbc13 tamper/between.py 6e72b92662185a56847cca235106bc354bd6a10e3e89a135b9ea8fa09cd8eb34 tamper/binary.py -9e1852d61d439181c42cb6d28656e9464a1dd5991269f000fb47e107f2f6f4f1 tamper/bluecoat.py -578e36fcf7d596574119ef75cbf1a83040913587a02855f0b6a7e684f9f9c8a5 tamper/chardoubleencode.py +f833cfbb53e6849ed1b3b554ec1c973f85e6d41ebd62f94f8e0dcf0ba5da2f49 tamper/bluecoat.py +69c7eb987dec666da227ee1024c31b89ad324a3f7cab287ada6dade7f51c8a36 tamper/chardoubleencode.py c7892bff56b2b85dfdf9f24c783c569edac57a3fd5a254cf4554987a374206c9 tamper/charencode.py 72c163ff0b4f79bdec07fbea3e75a2eaa8304881d35287eab8f03c25d06e99e0 tamper/charunicodeencode.py -50107854594fb13b4b95eed2ab8e66d2dd5470dd7d6b59c124ca766b1ec4b6ed tamper/charunicodeescape.py +249c938290c93df028a2b72762e6683be3ef6ea2bc334dd106af6d1a8048b97b tamper/charunicodeescape.py d0d8f2df2c29d81315a867ecb6baa9ca430e8f98d04f4df3879f2bcd697fac16 tamper/commalesslimit.py 1aee4e920b8ffa4a79b2ac9a42e2d7de13434970b3d1e0c6911c26bdd0c7b4e7 tamper/commalessmid.py ff8d05da2c5a123a231671c97ee80bb77b6631d7e5356d836cfe15ef212b73e5 tamper/commentbeforeparentheses.py 27f74b1c007713f753e0278bc056b09cd715c364847977962d6a198ecefa14ff tamper/concat2concatws.py -b5a5ba94a78cf83b35cdb0b08d9d69dbf1f33c07cc5152c560ae5aee54a4c066 tamper/decentities.py +4cc9f6d319fbf3b60de4b9a487f9630e95cfef0ebf7749b623526b91510668a5 tamper/decentities.py 1d6bcc5ffe235840370cd9738b5e8067f8b24e8c0e2bb629d330a7e5c379328a tamper/dunion.py -99c59e6fd7cafc9238c53e037eff457823854eef7cb0c5ea05941e0223229209 tamper/equaltolike.py -b3940e8d029150a81f17a2da1141928c31b6abb9ade3672d093051e310439995 tamper/equaltorlike.py +ab455ab2d7bf89e2d283799841556e2b87c53bd288aca88f2d9f1ea5b9c39cb8 tamper/equaltolike.py +c686219f6e1b22be654792ead82c55947c11dc55901db6173fbc9821b6da625d tamper/equaltorlike.py d528e74ae7c9fc0cd45369046d835a8f1e6f9252eeef6d84d9978d7e329ab35f tamper/escapequotes.py 0694f202a4f57e0a5c4d5aa72eee121b6f344d4e03692d9e267e2212abed719c tamper/greatest.py 89c2606da517d063f5a898a33d5bfd8737eef837552fc1127cea512ab82d0ea5 tamper/halfversionedmorekeywords.py -f0a7b635061385a3bf399cc51faf4d5e10694266aaa21fba557ca655c00a09bc tamper/hex2char.py -9096cbf2283137d592408325347f46866fd139966c946f8ba1ea61826472d0bb tamper/hexentities.py -3e518ace6940d54e8844c83781756e85d5670c53dfac0a092c4ee36cd5111885 tamper/htmlencode.py +76475815dedf1b56a542abdbad3f50f26f9b402775b6d475ba3b8ce64dede022 tamper/hex2char.py +731e7ab9996dbe701d5a4971540c92245d204c11bf00efcb905bb27f3269e97b tamper/hexentities.py +7324f520834d6072896df56802dca416ef66c175c339ed498708144bb51d193d tamper/htmlencode.py d05dafb86e82807e75bb8f54dcd6afbb4a08ba3b83b35562fee7f7022a75dbd7 tamper/if2case.py 55092820a856f583cf1b661001b60216886d172cb7d0008920bf4ab3df88aff0 tamper/ifnull2casewhenisnull.py eeda2b2fd54a4aa5fcf5630f8bfae43e0a38a840ae908e2f6b0878959067413c tamper/ifnull2ifisnull.py @@ -535,7 +535,7 @@ b533f576b260f485ebb70566c520979608d9f1790aa2811ce8194970b63e0d96 tamper/modsecu 687f531696809452a37f631cdb201267b04cb83b34a847aec507aca04e2ec305 tamper/ord2ascii.py 07cca753862dc9a2379aea23823d71ad6f4f6716a220e01792467549f8bde95a tamper/overlongutf8more.py b17748d63b763a7bfd2188f44145345507ce71e1b46f29d747132da5c56d7ed0 tamper/overlongutf8.py -dea9ab017cc4bde6f61f95a4f400ecba441525ff2d2dba886a2bf3ecdc1af605 tamper/percentage.py +88393d8062c76e402b811872a335db92b457aeca906835c751274b714def9e7e tamper/percentage.py 5437bc272398173c997d7b156dac1606dcde30421923bfc8f744d3668441d79e tamper/plus2concat.py 3cec7391b8b586474455ef4b089a27c67406ba02f91698647bb113c291f38692 tamper/plus2fnconcat.py f5e2cccbe669b732c0b8aaa56c16522fd579168ff61a92d31f94c6970070dfe0 tamper/randomcase.py @@ -554,10 +554,10 @@ cd972178ac4464c6692939c347a03a8c1f3f5dae9d3ef83ae82328fa542b7f49 tamper/space2m 0a3bc5380bddbfddfd32ce0a353f1abf57894f03262503c4f6e88748ae4a7f58 tamper/space2mysqldash.py ef090bed1c71b5d6cd6422748799236dbdadbc70593a7b8ccb26ad07c7a76946 tamper/space2plus.py 93d1cf1f6fb977356c4c8dc2d7784d4564b8da3d9f16e8253f957f80af2491f3 tamper/space2randomblank.py -6769cbe7b42265ff257a49e17e894bc19ff805802e19f27d57c07a212de70a11 tamper/sp_password.py +477ae0f9e3fe48b2fe5ced7b525b05a8e1db66963ff19dbb38dc810443dece57 tamper/sp_password.py 8e52309b893770bce57215fd3bf42d53d7f0d164690b4121b598126cbaaf6bc3 tamper/substring2leftright.py -d4b29c9a47961430dd0a24c22f8fe2968374ca5b0611e8b2837481c8d77672bf tamper/symboliclogical.py -c442ec7bb6676bdc58447fa54c719a9322b1728ba96c2358081a73fa8a4612ff tamper/unionalltounion.py +4b0dc71cef8daa67bcd54059e2a488340da9d64b5b2f848b2e2eff8972fc1649 tamper/symboliclogical.py +dcdeed9ee285e63cf06baf8347e3db7f210ef25a63869bab78ce1ec6898ae191 tamper/unionalltounion.py 9ebf67b9ce10b338edc3e804111abe56158fa0a69e53aacdd0ffa0e0b6af1f70 tamper/unmagicquotes.py 67a83f8b6e99e9bb3344ad6f403e1d784cf9d3f3b7e8e40053cf3181fabe47fa tamper/uppercase.py 3e54d7f98ca75181e6b16aa306d5a5f5f0dce857d5b3e6ce5a07d501f5d915aa tamper/varnish.py diff --git a/lib/controller/checks.py b/lib/controller/checks.py index 0181e91731..328b457a8a 100644 --- a/lib/controller/checks.py +++ b/lib/controller/checks.py @@ -1238,7 +1238,7 @@ def checkDynamicContent(firstPage, secondPage): kb.heavilyDynamic = True secondPage, _, _ = Request.queryPage(content=True) - findDynamicContent(firstPage, secondPage) + findDynamicContent(firstPage, secondPage, merge=True) def checkStability(): """ diff --git a/lib/core/agent.py b/lib/core/agent.py index be235b7447..9c109238ac 100644 --- a/lib/core/agent.py +++ b/lib/core/agent.py @@ -1229,6 +1229,9 @@ def addPayloadDelimiters(self, value): def removePayloadDelimiters(self, value): """ Removes payload delimiters from inside the input string + + >>> agent.removePayloadDelimiters(agent.addPayloadDelimiters("1 AND 1=1")) == "1 AND 1=1" + True """ return value.replace(PAYLOAD_DELIMITER, "") if value else value @@ -1236,6 +1239,9 @@ def removePayloadDelimiters(self, value): def extractPayload(self, value): """ Extracts payload from inside of the input string + + >>> agent.extractPayload("prefix" + agent.addPayloadDelimiters("1 AND 1=1") + "suffix") == "1 AND 1=1" + True """ _ = re.escape(PAYLOAD_DELIMITER) diff --git a/lib/core/common.py b/lib/core/common.py index efbe262ec4..e486a6fe14 100644 --- a/lib/core/common.py +++ b/lib/core/common.py @@ -3237,11 +3237,15 @@ def aliasToDbmsEnum(dbms): return retVal -def findDynamicContent(firstPage, secondPage): +def findDynamicContent(firstPage, secondPage, merge=False): """ This function checks if the provided pages have dynamic content. If they are dynamic, proper markings will be made + Note: with merge=True the newly found markings are accumulated into the + existing ones (e.g. when refining across multiple original-page samples) + instead of replacing them + >>> findDynamicContent("Lorem ipsum dolor sit amet, congue tation referrentur ei sed. Ne nec legimus habemus recusabo, natum reque et per. Facer tritani reprehendunt eos id, modus constituam est te. Usu sumo indoctum ad, pri paulo molestiae complectitur no.", "Lorem ipsum dolor sit amet, congue tation referrentur ei sed. Ne nec legimus habemus recusabo, natum reque et per. Facer tritani reprehendunt eos id, modus constituam est te. Usu sumo indoctum ad, pri paulo molestiae complectitur no.") >>> kb.dynamicMarkings [('natum reque et per. ', 'Facer tritani repreh')] @@ -3254,7 +3258,9 @@ def findDynamicContent(firstPage, secondPage): singleTimeLogMessage(infoMsg) blocks = list(SequenceMatcher(None, firstPage, secondPage).get_matching_blocks()) - kb.dynamicMarkings = [] + + if not merge: + kb.dynamicMarkings = [] # Removing too small matching blocks for block in blocks[:]: @@ -3292,7 +3298,9 @@ def findDynamicContent(firstPage, secondPage): suffix = trimAlphaNum(suffix) break - kb.dynamicMarkings.append((prefix if prefix else None, suffix if suffix else None)) + marking = (prefix if prefix else None, suffix if suffix else None) + if marking not in kb.dynamicMarkings: # Note: avoiding duplicates (e.g. when accumulating markings across samples) + kb.dynamicMarkings.append(marking) if len(kb.dynamicMarkings) > 0: infoMsg = "dynamic content marked for removal (%d region%s)" % (len(kb.dynamicMarkings), 's' if len(kb.dynamicMarkings) > 1 else '') @@ -4122,6 +4130,9 @@ def intersect(containerA, containerB, lowerCase=False): def decodeStringEscape(value): """ Decodes escaped string values (e.g. "\\t" -> "\t") + + >>> decodeStringEscape("a" + chr(92) + "tb") == "a" + chr(9) + "b" + True """ retVal = value @@ -4136,6 +4147,9 @@ def decodeStringEscape(value): def encodeStringEscape(value): """ Encodes escaped string values (e.g. "\t" -> "\\t") + + >>> encodeStringEscape("a" + chr(9) + "b") == "a" + chr(92) + "tb" + True """ retVal = value diff --git a/lib/core/option.py b/lib/core/option.py index 0649854d21..5f23a1aea5 100644 --- a/lib/core/option.py +++ b/lib/core/option.py @@ -2071,7 +2071,7 @@ def _setKnowledgeBaseAttributes(flushAll=True): kb.cache = AttribDict() kb.cache.addrinfo = {} kb.cache.content = LRUDict(capacity=16) - kb.cache.comparison = {} + kb.cache.comparison = LRUDict(capacity=256) kb.cache.encoding = LRUDict(capacity=256) kb.cache.alphaBoundaries = None kb.cache.hashRegex = None diff --git a/lib/core/settings.py b/lib/core/settings.py index 9acb384895..a399681809 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -20,7 +20,7 @@ from thirdparty import six # sqlmap version (...) -VERSION = "1.10.6.83" +VERSION = "1.10.6.95" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) diff --git a/lib/techniques/blind/inference.py b/lib/techniques/blind/inference.py index d9207e2b10..42c20f6868 100644 --- a/lib/techniques/blind/inference.py +++ b/lib/techniques/blind/inference.py @@ -152,6 +152,8 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None lastChar = 0 elif conf.lastChar is not None and (isinstance(conf.lastChar, int) or (hasattr(conf.lastChar, "isdigit") and conf.lastChar.isdigit())): lastChar = int(conf.lastChar) + if kb.fileReadMode: # Note: file content is retrieved hex-encoded (2 chars per byte), mirroring the firstChar handling above + lastChar <<= 1 elif hasattr(lastChar, "isdigit") and lastChar.isdigit() or isinstance(lastChar, int): lastChar = int(lastChar) else: diff --git a/lib/utils/safe2bin.py b/lib/utils/safe2bin.py index 35d0a77cba..b5a93b4f72 100644 --- a/lib/utils/safe2bin.py +++ b/lib/utils/safe2bin.py @@ -74,6 +74,11 @@ def safecharencode(value): def safechardecode(value, binary=False): """ Reverse function to safecharencode + + >>> safechardecode(u'test123') == u'test123' + True + >>> safechardecode(safecharencode(u'test\x01\x02\xaf')) == u'test\x01\x02\xaf' + True """ retVal = value diff --git a/lib/utils/search.py b/lib/utils/search.py index 985226891f..4e98a12f53 100644 --- a/lib/utils/search.py +++ b/lib/utils/search.py @@ -11,7 +11,6 @@ from lib.core.common import getSafeExString from lib.core.common import popValue from lib.core.common import pushValue -from lib.core.common import readInput from lib.core.common import urlencode from lib.core.convert import getBytes from lib.core.convert import getUnicode @@ -24,7 +23,6 @@ from lib.core.enums import REDIRECTION from lib.core.exception import SqlmapBaseException from lib.core.exception import SqlmapConnectionException -from lib.core.exception import SqlmapUserQuitException from lib.core.settings import BING_REGEX from lib.core.settings import DUCKDUCKGO_REGEX from lib.core.settings import DUMMY_SEARCH_USER_AGENT @@ -37,152 +35,102 @@ from thirdparty.six.moves import urllib as _urllib from thirdparty.socks import socks -def _search(dork): +def _fetch(url, headers, data=None): """ - This method performs the effective search on Google providing - the google dork and the Google session cookie + Fetches and returns the (decoded) content of a search engine results page + (or None in case of a connection issue) """ - if not dork: - return None - - page = None - data = None - requestHeaders = {} - responseHeaders = {} - - requestHeaders[HTTP_HEADER.USER_AGENT] = dict(conf.httpHeaders).get(HTTP_HEADER.USER_AGENT, DUMMY_SEARCH_USER_AGENT) - requestHeaders[HTTP_HEADER.ACCEPT_ENCODING] = HTTP_ACCEPT_ENCODING_HEADER_VALUE - requestHeaders[HTTP_HEADER.COOKIE] = GOOGLE_CONSENT_COOKIE + retVal = None try: - req = _urllib.request.Request("https://www.google.com/ncr", headers=requestHeaders) + req = _urllib.request.Request(url, data=getBytes(data) if data else None, headers=headers) conn = _urllib.request.urlopen(req) - except Exception as ex: - errMsg = "unable to connect to Google ('%s')" % getSafeExString(ex) - raise SqlmapConnectionException(errMsg) - gpage = conf.googlePage if conf.googlePage > 1 else 1 - logger.info("using search result page #%d" % gpage) - - url = "https://www.google.com/search?" # NOTE: if consent fails, try to use the "http://" - url += "q=%s&" % urlencode(dork, convall=True) - url += "num=100&hl=en&complete=0&safe=off&filter=0&btnG=Search" - url += "&start=%d" % ((gpage - 1) * 100) - - try: - req = _urllib.request.Request(url, headers=requestHeaders) - conn = _urllib.request.urlopen(req) - - requestMsg = "HTTP request:\nGET %s" % url + requestMsg = "HTTP request:\n%s %s" % ("POST" if data else "GET", url) requestMsg += " %s" % _http_client.HTTPConnection._http_vsn_str logger.log(CUSTOM_LOGGING.TRAFFIC_OUT, requestMsg) page = conn.read() - code = conn.code - status = conn.msg responseHeaders = conn.info() - responseMsg = "HTTP response (%s - %d):\n" % (status, code) - + responseMsg = "HTTP response (%s - %d):\n" % (conn.msg, conn.code) if conf.verbose <= 4: responseMsg += getUnicode(responseHeaders, UNICODE_ENCODING) elif conf.verbose > 4: responseMsg += "%s\n%s\n" % (responseHeaders, page) - logger.log(CUSTOM_LOGGING.TRAFFIC_IN, responseMsg) + + page = decodePage(page, responseHeaders.get(HTTP_HEADER.CONTENT_ENCODING), responseHeaders.get(HTTP_HEADER.CONTENT_TYPE)) + retVal = getUnicode(page) # Note: if decodePage call fails (Issue #4202) except _urllib.error.HTTPError as ex: try: - page = ex.read() - responseHeaders = ex.info() - except Exception as _: - warnMsg = "problem occurred while trying to get " - warnMsg += "an error page information (%s)" % getSafeExString(_) - logger.critical(warnMsg) - return None + retVal = getUnicode(ex.read()) + except Exception: + pass except (_urllib.error.URLError, _http_client.error, socket.error, socket.timeout, socks.ProxyError): - errMsg = "unable to connect to Google" - raise SqlmapConnectionException(errMsg) + pass - page = decodePage(page, responseHeaders.get(HTTP_HEADER.CONTENT_ENCODING), responseHeaders.get(HTTP_HEADER.CONTENT_TYPE)) + return retVal - page = getUnicode(page) # Note: if decodePage call fails (Issue #4202) +def _search(dork): + """ + This method performs the effective search using the provided dork, + trying the available search engines in order of (current) scraping + reliability and returning the results of the first one that yields any + (so that the failure of a single engine does not break the feature) + """ - retVal = [_urllib.parse.unquote(match.group(1) or match.group(2)) for match in re.finditer(GOOGLE_REGEX, page, re.I)] + if not dork: + return None - if not retVal and "detected unusual traffic" in page: - warnMsg = "Google has detected 'unusual' traffic from " - warnMsg += "used IP address disabling further searches" + retVal = [] + seen = set() - if conf.proxyList: + requestHeaders = { + HTTP_HEADER.USER_AGENT: dict(conf.httpHeaders).get(HTTP_HEADER.USER_AGENT, DUMMY_SEARCH_USER_AGENT), + HTTP_HEADER.ACCEPT_ENCODING: HTTP_ACCEPT_ENCODING_HEADER_VALUE, + HTTP_HEADER.COOKIE: GOOGLE_CONSENT_COOKIE, + } + + gpage = conf.googlePage if conf.googlePage > 1 else 1 + logger.info("using search result page #%d" % gpage) + + encoded = urlencode(dork, convall=True) + + # Note: (name, url, POST data, regex, regex flags, match->link). Ordered by current scraping reliability; tried in turn until one yields results (DuckDuckGo currently being the only consistently scrapeable one) + engines = ( + ("DuckDuckGo", "https://html.duckduckgo.com/html/", "q=%s&s=%d" % (encoded, (gpage - 1) * 30), DUCKDUCKGO_REGEX, re.I | re.S, lambda match: match.group(1).replace("&", "&")), + ("Bing", "https://www.bing.com/search?q=%s&first=%d" % (encoded, (gpage - 1) * 10 + 1), None, BING_REGEX, re.I | re.S, lambda match: match.group(1)), + ("Google", "https://www.google.com/search?q=%s&num=100&hl=en&complete=0&safe=off&filter=0&btnG=Search&start=%d" % (encoded, (gpage - 1) * 100), None, GOOGLE_REGEX, re.I, lambda match: match.group(1) or match.group(2)), + ) + + for name, url, data, regex, flags, extract in engines: + page = _fetch(url, requestHeaders, data) + + if not page: + continue + + count = 0 + for match in re.finditer(regex, page, flags): + link = _urllib.parse.unquote(extract(match)) + if link and link not in seen: + seen.add(link) + retVal.append(link) + count += 1 + + if count: + logger.info("found %d usable link%s using %s" % (count, 's' if count != 1 else "", name)) + break # Note: stop at the first engine that actually returns results (others are only fallbacks) + + # Note: switch proxy (if available) when an abuse/captcha page was served (instead of pointlessly falling through to the next engine from the same blocked IP) + if conf.proxyList and (("detected unusual traffic" in page) or ("issue with the Tor Exit Node you are currently using" in page)): + warnMsg = "%s has detected 'unusual' traffic from the used IP address" % name raise SqlmapBaseException(warnMsg) - else: - logger.critical(warnMsg) if not retVal: - message = "no usable links found. What do you want to do?" - message += "\n[1] (re)try with DuckDuckGo (default)" - message += "\n[2] (re)try with Bing" - message += "\n[3] quit" - choice = readInput(message, default='1') - - if choice == '3': - raise SqlmapUserQuitException - elif choice == '2': - url = "https://www.bing.com/search?q=%s&first=%d" % (urlencode(dork, convall=True), (gpage - 1) * 10 + 1) - regex = BING_REGEX - else: - url = "https://html.duckduckgo.com/html/" - data = "q=%s&s=%d" % (urlencode(dork, convall=True), (gpage - 1) * 30) - regex = DUCKDUCKGO_REGEX - - try: - req = _urllib.request.Request(url, data=getBytes(data), headers=requestHeaders) - conn = _urllib.request.urlopen(req) - - requestMsg = "HTTP request:\nGET %s" % url - requestMsg += " %s" % _http_client.HTTPConnection._http_vsn_str - logger.log(CUSTOM_LOGGING.TRAFFIC_OUT, requestMsg) - - page = conn.read() - code = conn.code - status = conn.msg - responseHeaders = conn.info() - page = decodePage(page, responseHeaders.get("Content-Encoding"), responseHeaders.get("Content-Type")) - - responseMsg = "HTTP response (%s - %d):\n" % (status, code) - - if conf.verbose <= 4: - responseMsg += getUnicode(responseHeaders, UNICODE_ENCODING) - elif conf.verbose > 4: - responseMsg += "%s\n%s\n" % (responseHeaders, page) - - logger.log(CUSTOM_LOGGING.TRAFFIC_IN, responseMsg) - except _urllib.error.HTTPError as ex: - try: - page = ex.read() - page = decodePage(page, ex.headers.get("Content-Encoding"), ex.headers.get("Content-Type")) - except socket.timeout: - warnMsg = "connection timed out while trying " - warnMsg += "to get error page information (%d)" % ex.code - logger.critical(warnMsg) - return None - except: - errMsg = "unable to connect" - raise SqlmapConnectionException(errMsg) - - page = getUnicode(page) # Note: if decodePage call fails (Issue #4202) - - retVal = [_urllib.parse.unquote(match.group(1).replace("&", "&")) for match in re.finditer(regex, page, re.I | re.S)] - - if not retVal and "issue with the Tor Exit Node you are currently using" in page: - warnMsg = "DuckDuckGo has detected 'unusual' traffic from " - warnMsg += "used (Tor) IP address" - - if conf.proxyList: - raise SqlmapBaseException(warnMsg) - else: - logger.critical(warnMsg) + warnMsg = "no usable links found (search engines might be blocking the used IP address)" + logger.critical(warnMsg) return retVal @@ -206,6 +154,7 @@ def search(dork): return search(dork) else: raise + finally: kb.choices.redirect = popValue() diff --git a/lib/utils/sqlalchemy.py b/lib/utils/sqlalchemy.py index 7506b42a79..1d4dccc2c0 100644 --- a/lib/utils/sqlalchemy.py +++ b/lib/utils/sqlalchemy.py @@ -122,6 +122,8 @@ def execute(self, query): try: self.cursor = self.connector.execute(query) + if hasattr(self.connector, "commit"): # Note: SQLAlchemy 2.0+ dropped implicit autocommit (otherwise DML changes - e.g. via --sql-query - would be silently lost) + self.connector.commit() retVal = True except (_sqlalchemy.exc.OperationalError, _sqlalchemy.exc.ProgrammingError) as ex: logger.log(logging.WARN if conf.dbmsHandler else logging.DEBUG, "(remote) %s" % getSafeExString(ex)) diff --git a/lib/utils/xrange.py b/lib/utils/xrange.py index 1a911b567e..19aa9713b2 100644 --- a/lib/utils/xrange.py +++ b/lib/utils/xrange.py @@ -26,6 +26,8 @@ class xrange(object): True >>> list(xrange(0, 7, 2)) == list(range(0, 7, 2)) True + >>> list(xrange(8, 0, -2)) == list(range(8, 0, -2)) + True >>> foobar = xrange(1, 10) >>> 7 in foobar True @@ -33,6 +35,12 @@ class xrange(object): False >>> foobar[0] 1 + >>> 6 in xrange(8, 0, -2) + True + >>> 0 in xrange(8, 0, -2) + False + >>> xrange(0, 10, 2).index(4) + 2 """ __slots__ = ['_slice'] @@ -71,10 +79,17 @@ def __len__(self): return self._len() def _len(self): - return max(0, 1 + int((self.stop - 1 - self.start) // self.step)) + if self.step > 0: + lo, hi, step = self.start, self.stop, self.step + else: # Note: normalizing for descending ranges (negative step) + lo, hi, step = self.stop, self.start, -self.step + return max(0, (hi - lo + step - 1) // step) def __contains__(self, value): - return (self.start <= value < self.stop) and (value - self.start) % self.step == 0 + if self.step > 0: + return self.start <= value < self.stop and (value - self.start) % self.step == 0 + else: + return self.stop < value <= self.start and (value - self.start) % self.step == 0 def __getitem__(self, index): if isinstance(index, slice): @@ -98,7 +113,7 @@ def _index(self, i): return self.start + self.step * i def index(self, i): - if self.start <= i < self.stop: - return i - self.start + if i in self: + return (i - self.start) // self.step # Note: also accounts for step != 1 (and descending ranges) else: raise ValueError("%d is not in list" % i) diff --git a/tamper/between.py b/tamper/between.py index 8e9538088f..5b289cb8a4 100644 --- a/tamper/between.py +++ b/tamper/between.py @@ -41,16 +41,16 @@ def tamper(payload, **kwargs): retVal = payload if payload: - match = re.search(r"(?i)(\b(AND|OR)\b\s+)(?!.*\b(AND|OR)\b)([^>]+?)\s*>\s*([^>]+)\s*\Z", payload) + match = re.search(r"(?i)(\b(AND|OR)\b\s+)(?!.*\b(AND|OR)\b)([^>]+?)\s*(?(?!=)\s*([^>]+)\s*\Z", payload) # Note: avoiding compound operators (e.g. >=, <>) if match: _ = "%s %s NOT BETWEEN 0 AND %s" % (match.group(2), match.group(4), match.group(5)) retVal = retVal.replace(match.group(0), _) else: - retVal = re.sub(r"\s*>\s*(\d+|'[^']+'|\w+\(\d+\))", r" NOT BETWEEN 0 AND \g<1>", payload) + retVal = re.sub(r"\s*(?(?!=)\s*(\d+|'[^']+'|\w+\(\d+\))", r" NOT BETWEEN 0 AND \g<1>", payload) if retVal == payload: - match = re.search(r"(?i)(\b(AND|OR)\b\s+)(?!.*\b(AND|OR)\b)([^=]+?)\s*=\s*([\w()]+)\s*", payload) + match = re.search(r"(?i)(\b(AND|OR)\b\s+)(?!.*\b(AND|OR)\b)([^=]+?)\s*(?!])=(?!=)\s*([\w()]+)\s*", payload) # Note: avoiding compound operators (e.g. >=, !=) if match: _ = "%s %s BETWEEN %s AND %s" % (match.group(2), match.group(4), match.group(5), match.group(5)) diff --git a/tamper/bluecoat.py b/tamper/bluecoat.py index 3ca4b8d4a3..7bfd30bd5c 100644 --- a/tamper/bluecoat.py +++ b/tamper/bluecoat.py @@ -44,7 +44,7 @@ def process(match): if payload: retVal = re.sub(r"\b(?P[A-Z_]+)(?=[^\w(]|\Z)", process, retVal) - retVal = re.sub(r"\s*=\s*", " LIKE ", retVal) + retVal = re.sub(r"\s*(?!=])=(?!=)\s*", " LIKE ", retVal) # Note: skipping compound operators (e.g. >=, <=, !=) retVal = retVal.replace("%09 ", "%09") return retVal diff --git a/tamper/chardoubleencode.py b/tamper/chardoubleencode.py index 4213421cb7..5f4639f786 100644 --- a/tamper/chardoubleencode.py +++ b/tamper/chardoubleencode.py @@ -9,7 +9,7 @@ from lib.core.enums import PRIORITY -__priority__ = PRIORITY.LOW +__priority__ = PRIORITY.LOWEST def dependencies(): pass diff --git a/tamper/charunicodeescape.py b/tamper/charunicodeescape.py index 80b600f9ca..0bc2624aec 100644 --- a/tamper/charunicodeescape.py +++ b/tamper/charunicodeescape.py @@ -9,7 +9,7 @@ from lib.core.enums import PRIORITY -__priority__ = PRIORITY.NORMAL +__priority__ = PRIORITY.LOWEST def tamper(payload, **kwargs): """ diff --git a/tamper/decentities.py b/tamper/decentities.py index 7ecb32cf4d..ee938ce504 100644 --- a/tamper/decentities.py +++ b/tamper/decentities.py @@ -7,7 +7,7 @@ from lib.core.enums import PRIORITY -__priority__ = PRIORITY.LOW +__priority__ = PRIORITY.LOWEST def dependencies(): pass diff --git a/tamper/equaltolike.py b/tamper/equaltolike.py index 9552dcb7a7..a4f8fa1c53 100644 --- a/tamper/equaltolike.py +++ b/tamper/equaltolike.py @@ -35,6 +35,6 @@ def tamper(payload, **kwargs): retVal = payload if payload: - retVal = re.sub(r"\s*=\s*", " LIKE ", retVal) + retVal = re.sub(r"\s*(?!=])=(?!=)\s*", " LIKE ", retVal) # Note: skipping compound operators (e.g. >=, <=, !=) return retVal diff --git a/tamper/equaltorlike.py b/tamper/equaltorlike.py index 0bad97d1fd..c617906a6e 100644 --- a/tamper/equaltorlike.py +++ b/tamper/equaltorlike.py @@ -32,6 +32,6 @@ def tamper(payload, **kwargs): retVal = payload if payload: - retVal = re.sub(r"\s*=\s*", " RLIKE ", retVal) + retVal = re.sub(r"\s*(?!=])=(?!=)\s*", " RLIKE ", retVal) # Note: skipping compound operators (e.g. >=, <=, !=) return retVal diff --git a/tamper/hex2char.py b/tamper/hex2char.py index 89bcc32c8c..f35709c12d 100644 --- a/tamper/hex2char.py +++ b/tamper/hex2char.py @@ -39,7 +39,7 @@ def tamper(payload, **kwargs): retVal = payload if payload: - for match in re.finditer(r"\b0x([0-9a-f]+)\b", retVal): + for match in re.finditer(r"(?i)\b0x([0-9a-f]+)\b", retVal): if len(match.group(1)) > 2: result = "CONCAT(%s)" % ','.join("CHAR(%d)" % _ for _ in getOrds(decodeHex(match.group(1)))) else: diff --git a/tamper/hexentities.py b/tamper/hexentities.py index 9b060673a0..b8f6813144 100644 --- a/tamper/hexentities.py +++ b/tamper/hexentities.py @@ -7,7 +7,7 @@ from lib.core.enums import PRIORITY -__priority__ = PRIORITY.LOW +__priority__ = PRIORITY.LOWEST def dependencies(): pass diff --git a/tamper/htmlencode.py b/tamper/htmlencode.py index ce09386be7..04810959a5 100644 --- a/tamper/htmlencode.py +++ b/tamper/htmlencode.py @@ -9,7 +9,7 @@ from lib.core.enums import PRIORITY -__priority__ = PRIORITY.LOW +__priority__ = PRIORITY.LOWEST def dependencies(): pass diff --git a/tamper/percentage.py b/tamper/percentage.py index 4f4da1f618..36c87dadb1 100644 --- a/tamper/percentage.py +++ b/tamper/percentage.py @@ -11,7 +11,7 @@ from lib.core.common import singleTimeWarnMessage from lib.core.enums import PRIORITY -__priority__ = PRIORITY.LOW +__priority__ = PRIORITY.LOWEST def dependencies(): singleTimeWarnMessage("tamper script '%s' is only meant to be run against ASP web applications" % os.path.basename(__file__).split(".")[0]) diff --git a/tamper/sp_password.py b/tamper/sp_password.py index 4efcc1c98e..95ec9dc489 100644 --- a/tamper/sp_password.py +++ b/tamper/sp_password.py @@ -7,7 +7,7 @@ from lib.core.enums import PRIORITY -__priority__ = PRIORITY.HIGH +__priority__ = PRIORITY.LOWEST def tamper(payload, **kwargs): """ diff --git a/tamper/symboliclogical.py b/tamper/symboliclogical.py index c7588aeb02..b255baeb16 100644 --- a/tamper/symboliclogical.py +++ b/tamper/symboliclogical.py @@ -9,7 +9,7 @@ from lib.core.enums import PRIORITY -__priority__ = PRIORITY.LOWEST +__priority__ = PRIORITY.HIGHEST def dependencies(): pass diff --git a/tamper/unionalltounion.py b/tamper/unionalltounion.py index 16e4ab7d47..c8007d67c1 100644 --- a/tamper/unionalltounion.py +++ b/tamper/unionalltounion.py @@ -5,6 +5,8 @@ See the file 'LICENSE' for copying permission """ +import re + from lib.core.enums import PRIORITY __priority__ = PRIORITY.HIGHEST @@ -20,4 +22,4 @@ def tamper(payload, **kwargs): '-1 UNION SELECT' """ - return payload.replace("UNION ALL SELECT", "UNION SELECT") if payload else payload + return re.sub(r"(?i)UNION\s+ALL\s+SELECT", "UNION SELECT", payload) if payload else payload