Merge pull request #54 from Snawoot/nv-driver-locator_scrapper_channel

nv-driver-locator: implemented new NvidiaDownloadsChannel "plugin"

win/tools/nv-driver-locator/README.md → tools/nv-driver-locator/README.md

@@ -11,6 +11,7 @@ nv-driver-locator is a tool for internal usage, which purpose is to notify about
 ## Requirements
 
 * Python 3.4+
+* `beautifulsoup4` package - required only when NvidiaDownloadsChannel is used.
 
 ## Overview
 
@@ -21,13 +22,14 @@ All scripts may be used both as standalone application and importable module. Fo
 * nv-driver-locator.py - main executable, intended to be run as cron job.
 * mailer.py - module with email routines and minimalistic email client for test purposes.
 * gfe\_get\_driver.py - GeForce Experience client library (and test util).
+* get\_nvidia\_downloads.py - Nvidia downloads site parser (and test util).
 
 ### Operation
 
 1. Cron job queries all configured channels.
 2. Program aggregates responses by hashing their's values covered by `key_components`. `key_components` is a list of JSON paths (represented by list too) specified in config file.
 3. Program queries DB if given hash has any match in database.
-4. If no match found and we have new instance all notifiers getting fired.
+4. If no match found and we have new instance, then all notifiers are getting fired.
 5. New record gets written into DB.
 
 ## Configuration example
@@ -44,6 +46,10 @@ All scripts may be used both as standalone application and importable module. Fo
         [
             "DriverAttributes",
             "Version"
+        ],
+        [
+            "DriverAttributes",
+            "Name"
         ]
     ],
     "channels": [
@@ -73,6 +79,114 @@ All scripts may be used both as standalone application and importable module. Fo
                 "notebook": true,
                 "beta": true
             }
+        },
+        {
+            "type": "nvidia_downloads",
+            "name": "linux beta",
+            "params": {
+                "os": "Linux_64",
+                "product": "GeForce",
+                "certlevel": "All"
+            }
+        },
+        {
+            "type": "nvidia_downloads",
+            "name": "linux stable",
+            "params": {
+                "os": "Linux_64",
+                "product": "GeForce",
+                "certlevel": "Certified"
+            }
+        },
+        {
+            "type": "nvidia_downloads",
+            "name": "downloads win stable",
+            "params": {
+                "os": "Windows10_64",
+                "product": "GeForce",
+                "certlevel": "Certified"
+            }
+        },
+        {
+            "type": "nvidia_downloads",
+            "name": "downloads win beta",
+            "params": {
+                "os": "Windows10_64",
+                "product": "GeForce",
+                "certlevel": "All"
+            }
+        },
+        {
+            "type": "nvidia_downloads",
+            "name": "downloads win notebook stable",
+            "params": {
+                "os": "Windows10_64",
+                "product": "GeForceMobile",
+                "certlevel": "Certified"
+            }
+        },
+        {
+            "type": "nvidia_downloads",
+            "name": "downloads win notebook beta",
+            "params": {
+                "os": "Windows10_64",
+                "product": "GeForceMobile",
+                "certlevel": "All"
+            }
+        },
+        {
+            "type": "nvidia_downloads",
+            "name": "linux quadro beta",
+            "params": {
+                "os": "Linux_64",
+                "product": "Quadro",
+                "certlevel": "All"
+            }
+        },
+        {
+            "type": "nvidia_downloads",
+            "name": "linux quadro stable",
+            "params": {
+                "os": "Linux_64",
+                "product": "Quadro",
+                "certlevel": "Certified"
+            }
+        },
+        {
+            "type": "nvidia_downloads",
+            "name": "downloads win quadro stable",
+            "params": {
+                "os": "Windows10_64",
+                "product": "Quadro",
+                "certlevel": "Certified"
+            }
+        },
+        {
+            "type": "nvidia_downloads",
+            "name": "downloads win quadro beta",
+            "params": {
+                "os": "Windows10_64",
+                "product": "Quadro",
+                "certlevel": "All"
+            }
+        },
+        {
+            "type": "nvidia_downloads",
+            "name": "downloads win quadro notebook stable",
+            "params": {
+                "os": "Windows10_64",
+                "product": "QuadroMobile",
+                "certlevel": "Certified"
+            }
+        },
+        {
+            "type": "nvidia_downloads",
+            "name": "downloads win quadro notebook beta",
+            "params": {
+                "os": "Windows10_64",
+                "product": "QuadroMobile",
+                "certlevel": "All"
+            }
         }
     ],
     "notifiers": [
@@ -138,6 +252,23 @@ Params:
 * `beta` - request Beta driver. Default: `false`
 * `dch` - request DCH driver. Default: `false` (request Standard Driver)
 
+#### NvidiaDownloadsChannel
+
+Parses Nvidia downloads site.
+
+Params:
+
+Type: `nvidia_downloads`
+
+Params:
+
+* `os` - OS family, version and bitness. Allowed values: `Linux_32`, `Linux_64`, `Windows7_32`, `Windows7_64`, `Windows10_32`, `Windows10_64`. Default: `Linux_64`.
+* `product` - product kind. Allowed values: `GeForce`, `GeForceMobile`, `Quadro`, `QuadroMobile`. Default: `GeForce`.
+* `certlevel` - driver certification level. Allowed values: `All` - any certification level, `Beta` - beta drivers, `Certified` - WHQL certified in Windows case and Nvidia certified in Linux case, `ODE` - Optimal Driver for Enterprise (Quadro driver), `QNF` - Quadro New Feature (Quadro driver). Default: `All`.
+* `driver_type` - driver type. Allowed values: `Standard`, `DCH`. At this moment DCH driver appears to exists only for some product families and only for Windows 10 x64. Default: `Standard`.
+* `lang` - driver language. Allowed values: `English`. Default: `English`.
+* `cuda_ver` - verson of CUDA Toolkit bundled with driver. Currently useless for covered product families. Default: `Nothing`.
+
 ### Notifiers
 
 #### CommandNotifier

tools/nv-driver-locator/get_nvidia_downloads.py

+#!/usr/bin/env python3
+
+import urllib.request
+import urllib.error
+import urllib.parse
+import codecs
+import enum
+from bs4 import BeautifulSoup
+
+USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64; rv:65.0) '\
+             'Gecko/20100101 Firefox/65.0'
+TIMEOUT = 10
+
+
+@enum.unique
+class OS(enum.Enum):
+    Linux_32 = 11
+    Linux_64 = 12
+    Windows7_32 = 18
+    Windows7_64 = 19
+    Windows10_32 = 56
+    Windows10_64 = 57
+
+    def __str__(self):
+        return self.name
+
+    def __contains__(self, e):
+        return e in self.__members__
+
+
+@enum.unique
+class CertLevel(enum.Enum):
+    All = ''
+    Beta = 0
+    Certified = 1
+    ODE = 2
+    QNF = 3
+
+    def __str__(self):
+        return self.name
+
+    def __contains__(self, e):
+        return e in self.__members__
+
+
+@enum.unique
+class Product(enum.Enum):
+    GeForce = (107, 879)
+    GeForceMobile = (111, 890)
+    Quadro = (73, 844)
+    QuadroMobile = (74, 875)
+
+    def __str__(self):
+        return self.name
+
+    def __contains__(self, e):
+        return e in self.__members__
+
+
+@enum.unique
+class DriverType(enum.Enum):
+    Standard = 0
+    DCH = 1
+
+    def __str__(self):
+        return self.name
+
+    def __contains__(self, e):
+        return e in self.__members__
+
+
+@enum.unique
+class DriverLanguage(enum.Enum):
+    English = 1
+
+    def __str__(self):
+        return self.name
+
+    def __contains__(self, e):
+        return e in self.__members__
+
+
+@enum.unique
+class CUDAToolkitVersion(enum.Enum):
+    Nothing = 0
+    v10_0 = 20
+
+    def __str__(self):
+        return self.name
+
+    def __contains__(self, e):
+        return e in self.__members__
+
+
+def parse_args():
+    import argparse
+
+    parser = argparse.ArgumentParser(
+        description="Retrieves info about latest NVIDIA drivers from "
+        "downloads site",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    parser.add_argument("-o", "--os",
+                        type=OS.__getitem__,
+                        choices=list(OS),
+                        default=OS.Linux_64,
+                        help="OS")
+    parser.add_argument("-p", "--product",
+                        type=Product.__getitem__,
+                        choices=list(Product),
+                        default=Product.GeForce,
+                        help="GPU Product type")
+    parser.add_argument("-c", "--certification-level",
+                        type=CertLevel.__getitem__,
+                        choices=list(CertLevel),
+                        default=CertLevel.All,
+                        help="driver certification level")
+    parser.add_argument("-D", "--dch",
+                        help="Query DCH driver instead of Standard driver",
+                        default=DriverType.Standard,
+                        const=DriverType.DCH,
+                        action="store_const")
+    parser.add_argument("-R", "--raw",
+                        help="Raw JSON output",
+                        action="store_true")
+    args = parser.parse_args()
+    return args
+
+
+def issue_request(query_obj):
+    ENDPOINT = 'https://www.nvidia.com/Download/processFind.aspx'
+    url = ENDPOINT + '?' + urllib.parse.urlencode(query_obj)
+    http_req = urllib.request.Request(
+        url,
+        data=None,
+        headers={
+            'User-Agent': USER_AGENT
+        }
+    )
+    with urllib.request.urlopen(http_req, None, TIMEOUT) as resp:
+        coding = resp.headers.get_content_charset()
+        coding = coding if coding is not None else 'utf-8-sig'
+        decoder = codecs.getreader(coding)(resp)
+        res = decoder.read()
+    return res
+
+
+def get_drivers(*,
+                os=OS.Linux_64,
+                product=Product.GeForce,
+                certlevel=CertLevel.All,
+                driver_type=DriverType.Standard,
+                lang=DriverLanguage.English,
+                cuda_ver=CUDAToolkitVersion.Nothing):
+    psid, pfid = product.value
+    query = {
+        'psid': psid,
+        'pfid': pfid,
+        'osid': os.value,
+        'lid': lang.value,
+        'whql': certlevel.value,
+        'lang': 'en-us',
+        'ctk': cuda_ver.value,
+    }
+    if os is OS.Windows10_64:
+        query['dtcid'] = driver_type.value
+    doc = issue_request(query)
+    soup = BeautifulSoup(doc, 'html.parser')
+    if soup.find(class_='contentBucketMainContent') is None:
+        return []
+    driverlistrows = list(
+        soup.find(class_='contentBucketMainContent')
+            .find_all('tr', id='driverList'))
+    if not driverlistrows:
+        return []
+    header = soup.find('td', class_='gridHeader').parent
+
+    def normalize_header(td):
+        return td.string.replace(' ', '').lower()
+
+    label_tuple = tuple(normalize_header(td) for td in header('td'))
+
+    def parse_content_td(td):
+        s = list(td.strings)
+        return max(s, key=len).strip() if s else ''
+
+    res = [dict(zip(label_tuple, (parse_content_td(td) for td in tr('td'))
+                    )) for tr in driverlistrows]
+    return res
+
+
+def main():
+    import pprint
+    args = parse_args()
+    pprint.pprint(get_drivers(os=args.os,
+                              product=args.product,
+                              certlevel=args.certification_level,
+                              driver_type=args.dch))
+
+
+if __name__ == '__main__':
+    main()

win/tools/nv-driver-locator/nv-driver-locator.json.sample → tools/nv-driver-locator/nv-driver-locator.json.sample

@@ -9,6 +9,10 @@
         [
             "DriverAttributes",
             "Version"
+        ],
+        [
+            "DriverAttributes",
+            "Name"
         ]
     ],
     "channels": [
@@ -38,6 +42,114 @@
                 "notebook": true,
                 "beta": true
             }
+        },
+        {
+            "type": "nvidia_downloads",
+            "name": "linux beta",
+            "params": {
+                "os": "Linux_64",
+                "product": "GeForce",
+                "certlevel": "All"
+            }
+        },
+        {
+            "type": "nvidia_downloads",
+            "name": "linux stable",
+            "params": {
+                "os": "Linux_64",
+                "product": "GeForce",
+                "certlevel": "Certified"
+            }
+        },
+        {
+            "type": "nvidia_downloads",
+            "name": "downloads win stable",
+            "params": {
+                "os": "Windows10_64",
+                "product": "GeForce",
+                "certlevel": "Certified"
+            }
+        },
+        {
+            "type": "nvidia_downloads",
+            "name": "downloads win beta",
+            "params": {
+                "os": "Windows10_64",
+                "product": "GeForce",
+                "certlevel": "All"
+            }
+        },
+        {
+            "type": "nvidia_downloads",
+            "name": "downloads win notebook stable",
+            "params": {
+                "os": "Windows10_64",
+                "product": "GeForceMobile",
+                "certlevel": "Certified"
+            }
+        },
+        {
+            "type": "nvidia_downloads",
+            "name": "downloads win notebook beta",
+            "params": {
+                "os": "Windows10_64",
+                "product": "GeForceMobile",
+                "certlevel": "All"
+            }
+        },
+        {
+            "type": "nvidia_downloads",
+            "name": "linux quadro beta",
+            "params": {
+                "os": "Linux_64",
+                "product": "Quadro",
+                "certlevel": "All"
+            }
+        },
+        {
+            "type": "nvidia_downloads",
+            "name": "linux quadro stable",
+            "params": {
+                "os": "Linux_64",
+                "product": "Quadro",
+                "certlevel": "Certified"
+            }
+        },
+        {
+            "type": "nvidia_downloads",
+            "name": "downloads win quadro stable",
+            "params": {
+                "os": "Windows10_64",
+                "product": "Quadro",
+                "certlevel": "Certified"
+            }
+        },
+        {
+            "type": "nvidia_downloads",
+            "name": "downloads win quadro beta",
+            "params": {
+                "os": "Windows10_64",
+                "product": "Quadro",
+                "certlevel": "All"
+            }
+        },
+        {
+            "type": "nvidia_downloads",
+            "name": "downloads win quadro notebook stable",
+            "params": {
+                "os": "Windows10_64",
+                "product": "QuadroMobile",
+                "certlevel": "Certified"
+            }
+        },
+        {
+            "type": "nvidia_downloads",
+            "name": "downloads win quadro notebook beta",
+            "params": {
+                "os": "Windows10_64",
+                "product": "QuadroMobile",
+                "certlevel": "All"
+            }
         }
     ],
     "notifiers": [

win/tools/nv-driver-locator/nv-driver-locator.py → tools/nv-driver-locator/nv-driver-locator.py

@@ -157,6 +157,43 @@ class GFEClientChannel(BaseChannel):
         return self._get_latest_driver(**self._kwargs)
 
 
+class NvidiaDownloadsChannel(BaseChannel):
+    def __init__(self, name, *,
+                 os="Linux_64",
+                 product="GeForce",
+                 certlevel="All",
+                 driver_type="Standard",
+                 lang="English",
+                 cuda_ver="Nothing"):
+        self.name = name
+        gnd = importlib.import_module('get_nvidia_downloads')
+        self._gnd = gnd
+        self._os = gnd.OS[os]
+        self._product = gnd.Product[product]
+        self._certlevel = gnd.CertLevel[certlevel]
+        self._driver_type = gnd.DriverType[driver_type]
+        self._lang = gnd.DriverLanguage[lang]
+        self._cuda_ver = gnd.CUDAToolkitVersion[cuda_ver]
+
+    def get_latest_driver(self):
+        drivers = self._gnd.get_drivers(os=self._os,
+                                        product=self._product,
+                                        certlevel=self._certlevel,
+                                        driver_type=self._driver_type,
+                                        lang=self._lang,
+                                        cuda_ver=self._cuda_ver)
+        if not drivers:
+            return None
+        latest = max(drivers, key=lambda d: tuple(d['version'].split('.')))
+        return {
+            'DriverAttributes': {
+                'Version': latest['version'],
+                'Name': latest['name'],
+                'NameLocalized': latest['name'],
+            }
+        }
+
+
 def parse_args():
     parser = argparse.ArgumentParser(
         description="Watches for GeForce experience driver updates for "
@@ -182,6 +219,7 @@ class DriverLocator:
     def _construct_channels(self, channels_config):
         channel_types = {
             'gfe_client': GFEClientChannel,
+            'nvidia_downloads': NvidiaDownloadsChannel,
         }
 
         channels = []