diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..91302ed2 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,27 @@ +# This workflow will do a clean install of node dependencies, cache/restore them, build the source code and run tests across different versions of node +# For more information see: https://github.com/marketplace/actions/setup-node-js-environment + +name: CI + +on: + push: + branches: [ master ] + pull_request: + branches: [] + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + node-version: [16.x, 18.x, 20.x] + # See supported Node.js release schedule at https://nodejs.org/en/about/releases/ + steps: + - uses: actions/checkout@v3 + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node-version }} + # cache: 'npm' + - run: npm install --ignore-scripts + - run: npm test diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 91fc0019..00000000 --- a/.travis.yml +++ /dev/null @@ -1,12 +0,0 @@ -language: node_js -node_js: - - 12 - - 14 - - node - -script: - - "npm test" - -notifications: - irc: "chat.freenode.net#ua-parser" - diff --git a/package.json b/package.json index b0d0d1a3..1f73caa8 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "uap-core", "description": "The regex file necessary to build language ports of Browserscope's user agent parser.", - "version": "0.16.0", + "version": "0.18.0", "maintainers": [ { "name": "Tobie Langel", @@ -25,7 +25,7 @@ } ], "devDependencies": { - "mocha": "^9.2.2", + "mocha": "^10.2.0", "safe-regex": "^2.1.1", "uap-ref-impl": "git+https://github.com/ua-parser/uap-ref-impl#master", "yamlparser": "^0.0.2" diff --git a/regexes.yaml b/regexes.yaml index a24a1c9c..a8f8e097 100644 --- a/regexes.yaml +++ b/regexes.yaml @@ -78,6 +78,9 @@ user_agent_parsers: # 'Mozilla/5.0 (Unknown; Linux x86_64) AppleWebKit/534.34 (KHTML, like Gecko) PingdomTMS/0.8.5 Safari/534.34' - regex: '(PingdomTMS)/(\d+)\.(\d+)\.(\d+)' family_replacement: 'PingdomBot' + # 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/61.0.3163.100 Chrome/61.0.3163.100 Safari/537.36 PingdomPageSpeed/1.0 (pingbot/2.0; +http://www.pingdom.com/)' + - regex: '(PingdomPageSpeed)/(\d+)\.(\d+)' + family_replacement: 'PingdomBot' # PTST / WebPageTest.org crawlers - regex: ' (PTST)/(\d+)(?:\.(\d+)|)$' @@ -217,11 +220,11 @@ user_agent_parsers: - regex: 'Mozilla.{1,200}Mobile.{1,100}(Phantom\/ios|android).(\d+)\.(\d+)\.(\d+)' family_replacement: 'Phantom' - # aspiegel.com spider (owned by Huawei) - - regex: 'Mozilla.{1,100}Mobile.{1,100}AspiegelBot' + # aspiegel.com spider (owned by Huawei, later called PetalBot) + - regex: 'Mozilla.{1,100}Mobile.{1,100}(AspiegelBot|PetalBot)' family_replacement: 'Spider' - - regex: 'AspiegelBot' + - regex: 'AspiegelBot|PetalBot' family_replacement: 'Spider' # Basilisk @@ -385,9 +388,13 @@ user_agent_parsers: # Edge Mobile - regex: 'Windows Phone .{0,200}(Edge)/(\d+)\.(\d+)' family_replacement: 'Edge Mobile' - - regex: '(EdgiOS|EdgA)/(\d+)\.(\d+)\.(\d+)\.(\d+)' + - regex: '(EdgiOS|EdgA)/(\d+)\.(\d+)\.(\d+)(?:\.(\d+)|)' family_replacement: 'Edge Mobile' + # Oculus Browser, should go before Samsung Internet + - regex: '(OculusBrowser)/(\d+)\.(\d+).0.0(?:\.([0-9\-]+)|)' + family_replacement: 'Oculus Browser' + # Samsung Internet (based on Chrome, but lacking some features) - regex: '(SamsungBrowser)/(\d+)\.(\d+)' family_replacement: 'Samsung Internet' @@ -921,7 +928,7 @@ user_agent_parsers: # AFTER THE EDGE CASES ABOVE! # AFTER IE11 # BEFORE all other IE - - regex: '(Firefox)/(\d+)\.(\d+)\.(\d+)' + - regex: '(Firefox)/(\d+)\.(\d+)(?:\.(\d+)|$)' - regex: '(Firefox)/(\d+)\.(\d+)(pre|[ab]\d+[a-z]*|)' @@ -932,7 +939,7 @@ user_agent_parsers: family_replacement: 'Python Requests' # headless user-agents - - regex: '\b(Windows-Update-Agent|WindowsPowerShell|Microsoft-CryptoAPI|SophosUpdateManager|SophosAgent|Debian APT-HTTP|Ubuntu APT-HTTP|libcurl-agent|libwww-perl|urlgrabber|curl|PycURL|Wget|wget2|aria2|Axel|OpenBSD ftp|lftp|jupdate|insomnia|fetch libfetch|akka-http|got|CloudCockpitBackend|ReactorNetty|axios|Jersey|Vert.x-WebClient|Apache-CXF|Go-CF-client|go-resty|AHC)(?:[ /](\d+)(?:\.(\d+)|)(?:\.(\d+)|)|)' + - regex: '\b(Windows-Update-Agent|WindowsPowerShell|Microsoft-CryptoAPI|SophosUpdateManager|SophosAgent|Debian APT-HTTP|Ubuntu APT-HTTP|libcurl-agent|libwww-perl|urlgrabber|curl|PycURL|Wget|wget2|aria2|Axel|OpenBSD ftp|lftp|jupdate|insomnia|fetch libfetch|akka-http|got|CloudCockpitBackend|ReactorNetty|axios|Jersey|Vert.x-WebClient|Apache-CXF|Go-CF-client|go-resty|AHC|HTTPie)(?:[ /](\d+)(?:\.(\d+)|)(?:\.(\d+)|)|)' # CloudFoundry - regex: '^(cf)\/(\d+)\.(\d+)\.(\S+)' @@ -961,7 +968,9 @@ user_agent_parsers: - regex: '(Python/3\.\d{1,3} aiohttp)/(\d+)\.(\d+)\.(\d+)' family_replacement: 'Python aiohttp' - - regex: '(Java)[/ ]?\d+\.(\d+)\.(\d+)[_-]*([a-zA-Z0-9]+|)' + - regex: '(Java)[/ ]?\d{1}\.(\d+)\.(\d+)[_-]*([a-zA-Z0-9]+|)' + + - regex: '(Java)[/ ]?(\d+)\.(\d+)\.(\d+)' # minio-go (https://github.com/minio/minio-go) - regex: '(minio-go)/v(\d+)\.(\d+)\.(\d+)' @@ -1077,8 +1086,8 @@ os_parsers: # generic HbbTV, hoping to catch manufacturer name (always after 2nd comma) and the first string that looks like a 2011-2019 year - regex: 'HbbTV/\d+\.\d+\.\d+ \(.{0,30}; ?([a-zA-Z]+) ?;.{0,30}(201[1-9]).{0,30}\)' - # aspiegel.com spider (owned by Huawei) - - regex: 'AspiegelBot' + # aspiegel.com spider (owned by Huawei, later renamed PetalBot) + - regex: 'AspiegelBot|PetalBot' os_replacement: 'Other' ########## @@ -1145,6 +1154,12 @@ os_parsers: - regex: '(android)\s(?:mobile\/)(\d+)(?:\.(\d+)(?:\.(\d+)|)|)' os_replacement: 'Android' + ########## + # Meta Quest + ########## + - regex: 'Quest' + os_replacement: 'Android' + ########## # Kindle Android ########## @@ -1595,6 +1610,29 @@ os_parsers: - regex: 'CFNetwork/.{0,100} Darwin/(21)\.\d+' os_replacement: 'iOS' os_v1_replacement: '15' + - regex: 'CFNetwork/.{0,100} Darwin/22\.0\.\d+' + os_replacement: 'iOS' + os_v1_replacement: '16' + os_v2_replacement: '0' + - regex: 'CFNetwork/.{0,100} Darwin/22\.1\.\d+' + os_replacement: 'iOS' + os_v1_replacement: '16' + os_v2_replacement: '1' + - regex: 'CFNetwork/.{0,100} Darwin/22\.2\.\d+' + os_replacement: 'iOS' + os_v1_replacement: '16' + os_v2_replacement: '2' + - regex: 'CFNetwork/.{0,100} Darwin/22\.3\.\d+' + os_replacement: 'iOS' + os_v1_replacement: '16' + os_v2_replacement: '3' + - regex: 'CFNetwork/.{0,100} Darwin/22\.4\.\d+' + os_replacement: 'iOS' + os_v1_replacement: '16' + os_v2_replacement: '4' + - regex: 'CFNetwork/.{0,100} Darwin/(22)\.\d+' + os_replacement: 'iOS' + os_v1_replacement: '16' - regex: 'CFNetwork/.{0,100} Darwin/' os_replacement: 'iOS' @@ -1823,11 +1861,11 @@ device_parsers: brand_replacement: 'Spider' # aspiegel.com spider (owned by Huawei) - - regex: 'Mozilla.{1,100}Mobile.{1,100}AspiegelBot' + - regex: 'Mozilla.{1,100}Mobile.{1,100}(AspiegelBot|PetalBot)' device_replacement: 'Spider' brand_replacement: 'Spider' model_replacement: 'Smartphone' - - regex: 'Mozilla.{0,200}AspiegelBot' + - regex: 'Mozilla.{0,200}(AspiegelBot|PetalBot)' device_replacement: 'Spider' brand_replacement: 'Spider' model_replacement: 'Desktop' @@ -3540,6 +3578,25 @@ device_parsers: brand_replacement: 'Meizu' model_replacement: '$1' + ######### + # Meta + # @ref: https://www.meta.com + ######### + - regex: 'Quest 2' + device_replacement: 'Quest' + brand_replacement: 'Meta' + model_replacement: 'Quest 2' + + - regex: 'Quest Pro' + device_replacement: 'Quest' + brand_replacement: 'Meta' + model_replacement: 'Quest Pro' + + - regex: 'Quest' + device_replacement: 'Quest' + brand_replacement: 'Meta' + model_replacement: 'Quest' + ######### # Micromax # @ref: http://www.micromaxinfo.com @@ -5549,7 +5606,7 @@ device_parsers: device_replacement: 'Motorola$2' brand_replacement: 'Motorola' model_replacement: '$2' - + ########## # nintendo diff --git a/test_resources/firefox_user_agent_strings.yaml b/test_resources/firefox_user_agent_strings.yaml index 391a8138..f8a1c884 100644 --- a/test_resources/firefox_user_agent_strings.yaml +++ b/test_resources/firefox_user_agent_strings.yaml @@ -1234,6 +1234,12 @@ test_cases: minor: '0' patch: '1' + - user_agent_string: 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:104.0) Gecko/20100101 Firefox/104.0' + family: 'Firefox' + major: '104' + minor: '0' + patch: + - user_agent_string: 'Mozilla/5.0 (X11; Linux armv7l; rv:2.1.1) Gecko/ Firefox/5.0.1' family: 'Firefox' major: '5' diff --git a/tests/test_device.yaml b/tests/test_device.yaml index bb43f59b..990c2208 100644 --- a/tests/test_device.yaml +++ b/tests/test_device.yaml @@ -145,6 +145,21 @@ test_cases: brand: 'Apple' model: 'iPod' + - user_agent_string: 'Mozilla/5.0 (X11; Linux x86_64; Quest) AppleWebKit/537.36 (KHTML, like Gecko) OculusBrowser/26.2.0.0.10 SamsungBrowser/4.0 Chrome/110.0.5481.192 VR Safari/537.36' + family: 'Quest' + brand: 'Meta' + model: 'Quest' + + - user_agent_string: 'Mozilla/5.0 (X11; Linux x86_64; Quest 2) AppleWebKit/537.36 (KHTML, like Gecko) OculusBrowser/26.2.0.0.10 SamsungBrowser/4.0 Chrome/110.0.5481.192 VR Safari/537.36' + family: 'Quest' + brand: 'Meta' + model: 'Quest 2' + + - user_agent_string: 'Mozilla/5.0 (X11; Linux x86_64; Quest Pro) AppleWebKit/537.36 (KHTML, like Gecko) OculusBrowser/26.2.0.0.10 SamsungBrowser/4.0 Chrome/110.0.5481.192 VR Safari/537.36' + family: 'Quest' + brand: 'Meta' + model: 'Quest Pro' + - user_agent_string: 'Mozilla/4.0 (compatible; Linux 2.6.10) NetFront/3.3 Kindle/1.0 (screen 600x800)' family: 'Kindle' brand: 'Amazon' @@ -80444,6 +80459,16 @@ test_cases: brand: 'Spider' model: 'Desktop' + - user_agent_string: 'Mozilla/5.0 (Linux; Android 7.0;) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; PetalBot;+https://webmaster.petalsearch.com/site/petalbot)' + family: 'Spider' + brand: 'Spider' + model: 'Smartphone' + + - user_agent_string: 'Mozilla/5.0 (compatible;PetalBot;+https://webmaster.petalsearch.com/site/petalbot)' + family: 'Spider' + brand: 'Spider' + model: 'Desktop' + - user_agent_string: 'Mozilla/5.0 (Linux; Android 7.0;) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; AspiegelBot)' family: 'Spider' brand: 'Spider' diff --git a/tests/test_os.yaml b/tests/test_os.yaml index 0e71e611..eafd1092 100644 --- a/tests/test_os.yaml +++ b/tests/test_os.yaml @@ -28,6 +28,13 @@ test_cases: patch: '2' patch_minor: + - user_agent_string: 'Mozilla/5.0 (X11; Linux x86_64; Quest 2) AppleWebKit/537.36 (KHTML, like Gecko) OculusBrowser/26.2.0.0.10 SamsungBrowser/4.0 Chrome/110.0.5481.192 VR Safari/537.36' + family: 'Android' + major: + minor: + patch: + patch_minor: + - user_agent_string: 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_3; en-us; Silk/1.1.0-80) AppleWebKit/533.16 (KHTML, like Gecko) Version/5.0 Safari/533.16 Silk-Accelerated=true' family: 'Android' major: @@ -2418,6 +2425,41 @@ test_cases: patch: patch_minor: + - user_agent_string: 'App/0 CFNetwork/1390 Darwin/22.0.0' + family: 'iOS' + major: '16' + minor: '0' + patch: + patch_minor: + + - user_agent_string: 'App/0 CFNetwork/1399 Darwin/22.1.0' + family: 'iOS' + major: '16' + minor: '1' + patch: + patch_minor: + + - user_agent_string: 'App/0 CFNetwork/1402 Darwin/22.2.0' + family: 'iOS' + major: '16' + minor: '2' + patch: + patch_minor: + + - user_agent_string: 'App/0 CFNetwork/1404.0.5 Darwin/22.3.0' + family: 'iOS' + major: '16' + minor: '3' + patch: + patch_minor: + + - user_agent_string: 'App/0 CFNetwork/1406.0.4 Darwin/22.4.0' + family: 'iOS' + major: '16' + minor: '4' + patch: + patch_minor: + - user_agent_string: 'Mozilla/5.0 (iPhone; CPU 7_0_6 like Mac OS X; en_GB) AppleWebKit (KHTML, like Gecko) Mobile [OKMagazine/4.0.1 (iOS/7.0.6)] [LiteApps]' family: 'iOS' major: '7' diff --git a/tests/test_ua.yaml b/tests/test_ua.yaml index b04999e7..2b2bee8c 100644 --- a/tests/test_ua.yaml +++ b/tests/test_ua.yaml @@ -83,6 +83,12 @@ test_cases: minor: '5' patch: '1' + - user_agent_string: 'Mozilla/5.0 (X11; Linux x86_64; Quest 2) AppleWebKit/537.36 (KHTML, like Gecko) OculusBrowser/26.2.0.0.10 SamsungBrowser/4.0 Chrome/110.0.5481.192 VR Safari/537.36' + family: 'Oculus Browser' + major: '26' + minor: '2' + patch: '10' + - user_agent_string: 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_3; en-us; Silk/1.1.0-80) AppleWebKit/533.16 (KHTML, like Gecko) Version/5.0 Safari/533.16 Silk-Accelerated=true' family: 'Amazon Silk' major: '1' @@ -551,6 +557,12 @@ test_cases: minor: '0' patch: '25' + - user_agent_string: 'Java/17.0.6' + family: 'Java' + major: '17' + minor: '0' + patch: '6' + - user_agent_string: 'Mozilla/5.0 (Linux; U; en-US) AppleWebKit/528.5+ (KHTML, like Gecko, Safari/528.5+) Version/4.0 Kindle/3.0 (screen 600x800; rotate)' family: 'Kindle' major: '3' @@ -959,6 +971,12 @@ test_cases: minor: '8' patch: '5' + - user_agent_string: 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/61.0.3163.100 Chrome/61.0.3163.100 Safari/537.36 PingdomPageSpeed/1.0 (pingbot/2.0; +http://www.pingdom.com/)' + family: 'PingdomBot' + major: '1' + minor: '0' + patch: + - user_agent_string: 'NewRelicPinger/1.0 (1025794)' family: 'NewRelicPingerBot' major: '1' @@ -6938,6 +6956,12 @@ test_cases: minor: '5' patch: '0' + - user_agent_string: 'Mozilla/5.0 (iPad; CPU OS 12_5_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0 EdgiOS/46.3.26 Mobile/15E148 Safari/605.1.15' + family: 'Edge Mobile' + major: '46' + minor: '3' + patch: '26' + - user_agent_string: 'Mozilla/5.0 (Linux; Android 8.1.0; Pixel Build/OPM4.171019.021.D1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.109 Mobile Safari/537.36 EdgA/42.0.0.2057' family: 'Edge Mobile' major: '42' @@ -8651,3 +8675,9 @@ test_cases: major: '2' minor: '7' patch: '6' + + - user_agent_string: 'HTTPie/3.2.1' + family: 'HTTPie' + major: '3' + minor: '2' + patch: '1'