Skip to content

N
nvidia-patch
Commit e56416e4 authored by Snawoot's avatar Snawoot Committed by GitHub
Browse files
Options

Merge pull request #338 from Snawoot/linux_nvidia_deb_channel 

Linux nvidia deb channel
parents 68eacc23 1207e650
Hide whitespace changes
Inline Side-by-side
Showing with 304 additions and 281 deletions
tools/nv-driver-locator/README.md
View file @ e56416e4
......@@ -35,232 +35,7 @@ All scripts may be used both as standalone application and importable module. Fo
## Configuration example
```json
{
"db": {
"type": "file",
"params": {
"workdir": "/var/lib/nv-driver-locator"
}
},
"key_components": [
[
"DriverAttributes",
"Version"
],
[
"DriverAttributes",
"Name"
],
[
"ChannelAttributes",
"OS"
]
],
"channels": [
{
"type": "gfe_client",
"name": "desktop defaults",
"params": {}
},
{
"type": "gfe_client",
"name": "desktop beta",
"params": {
"beta": true
}
},
{
"type": "gfe_client",
"name": "mobile",
"params": {
"notebook": true
}
},
{
"type": "gfe_client",
"name": "mobile beta",
"params": {
"notebook": true,
"beta": true
}
},
{
"type": "nvidia_downloads",
"name": "linux beta",
"params": {
"os": "Linux_64",
"product": "GeForce",
"certlevel": "All"
}
},
{
"type": "nvidia_downloads",
"name": "linux stable",
"params": {
"os": "Linux_64",
"product": "GeForce",
"certlevel": "Certified"
}
},
{
"type": "nvidia_downloads",
"name": "downloads win stable",
"params": {
"os": "Windows10_64",
"product": "GeForce",
"certlevel": "Certified"
}
},
{
"type": "nvidia_downloads",
"name": "downloads win beta",
"params": {
"os": "Windows10_64",
"product": "GeForce",
"certlevel": "All"
}
},
{
"type": "nvidia_downloads",
"name": "downloads win notebook stable",
"params": {
"os": "Windows10_64",
"product": "GeForceMobile",
"certlevel": "Certified"
}
},
{
"type": "nvidia_downloads",
"name": "downloads win notebook beta",
"params": {
"os": "Windows10_64",
"product": "GeForceMobile",
"certlevel": "All"
}
},
{
"type": "nvidia_downloads",
"name": "linux quadro beta",
"params": {
"os": "Linux_64",
"product": "Quadro",
"certlevel": "All"
}
},
{
"type": "nvidia_downloads",
"name": "linux quadro stable",
"params": {
"os": "Linux_64",
"product": "Quadro",
"certlevel": "Certified"
}
},
{
"type": "nvidia_downloads",
"name": "downloads win quadro stable",
"params": {
"os": "Windows10_64",
"product": "Quadro",
"certlevel": "Certified"
}
},
{
"type": "nvidia_downloads",
"name": "downloads win quadro beta",
"params": {
"os": "Windows10_64",
"product": "Quadro",
"certlevel": "All"
}
},
{
"type": "nvidia_downloads",
"name": "downloads win quadro notebook stable",
"params": {
"os": "Windows10_64",
"product": "QuadroMobile",
"certlevel": "Certified"
}
},
{
"type": "nvidia_downloads",
"name": "downloads win quadro notebook beta",
"params": {
"os": "Windows10_64",
"product": "QuadroMobile",
"certlevel": "All"
}
},
{
"type": "nvidia_downloads",
"name": "downloads win server quadro certified",
"params": {
"os": "WindowsServer2012R2_64",
"product": "Quadro",
"certlevel": "Certified"
}
},
{
"type": "nvidia_downloads",
"name": "downloads win server 2019 quadro certified",
"params": {
"os": "WindowsServer2019",
"product": "Quadro",
"certlevel": "Certified"
}
},
{
"type": "cuda_downloads",
"name": "cuda toolkit tracker",
"params": {}
},
{
"type": "vulkan_beta",
"name": "vulkan beta windows",
"params": {
"os": "Windows"
}
},
{
"type": "vulkan_beta",
"name": "vulkan beta linux",
"params": {
"os": "Linux"
}
}
],
"notifiers": [
{
"type": "email",
"name": "my email",
"params": {
"from_addr": "notify-bot@gmail.com",
"to_addrs": [
"recepient1@domain1.tld",
"recepient2@domain2.tld"
],
"host": "smtp.google.com",
"use_starttls": true,
"login": "notify-bot",
"password": "MyGoodPass"
}
},
{
"type": "command",
"name": "sample command",
"params": {
"timeout": 10.0,
"cmdline": [
"cat",
"-"
]
}
}
]
}
```
See [nv-driver-locator.json.sample](nv-driver-locator.json.sample).
## Components Reference
......@@ -329,7 +104,19 @@ Type: `vulkan_beta`
Params:
* `os` - OS family. Allowed values: `Linux`, `Windows`. Default: `Linux`.
* `timeout` - allowed delay in seconds for each network operation. Default: `10.0`
#### DebRepoChannel
Parses Packages or Packages.gz file of deb package repository and extracts versions information.
Type: `deb_packages`
Params:
* `url` - Location of `Packages` or `Packages.gz` file.
* `pkg_pattern` - regexp which defines package name pattern.
* `driver_name` - value returned as `DriverAttributes.Name` for proper aggregation by Hasher. Default: `"Linux x64 (AMD64/EM64T) Display Driver"`
* `timeout` - allowed delay in seconds for each network operation. Default: `10.0`
### Notifiers
......
tools/nv-driver-locator/get_deb_drivers.py 0 → 100755
View file @ e56416e4
#!/usr/bin/env python3
import urllib.request
import urllib.error
import json
import posixpath
import codecs
import gzip
import sys
from contextlib import contextmanager
import itertools
import string
import codecs
import pprint
import re
import collections
USER_AGENT = 'Debian APT-HTTP/1.3 (1.6.6)'
DEFAULT_REPO = "https://developer.download.nvidia.com/"\
"compute/cuda/repos/ubuntu1804/x86_64/Packages.gz"
DEFAULT_REGEX = '^cuda-drivers$'
ENCODING = 'utf-8-sig'
DriverEntry = collections.namedtuple('DriverEntry', ('name', 'version'))
def upstream_version(version):
epoch, delim, tail = version.partition(':')
version = tail if delim else epoch
return version.partition('-')[0]
@contextmanager
def packages_reader(url, timeout):
http_req = urllib.request.Request(
url,
data=None,
headers={
'User-Agent': USER_AGENT
}
)
with urllib.request.urlopen(http_req, None, timeout) as resp:
if url.endswith('.gz') or resp.headers.get('Content-Type', '') == 'application/x-gzip':
with gzip.GzipFile(fileobj=resp) as reader:
yield codecs.getreader(ENCODING)(reader)
else:
yield codecs.getreader(ENCODING)(resp)
def parse_packages(reader):
for k, g in itertools.groupby(reader, lambda s: bool(s.strip())):
if not k:
continue
pkg = dict()
current_key = None
current_val = None
for line in g:
if line[0] in string.whitespace:
# Continuation
if current_key is None:
continue
current_val += line.rstrip()
else:
# New field
if current_key is not None:
pkg[current_key.lower()] = current_val
current_key, _, current_val = line.partition(':')
current_val = current_val.strip()
if current_key is not None:
pkg[current_key.lower()] = current_val
if 'package' in pkg and 'version' in pkg:
yield pkg
def _get_deb_versions(*, url=DEFAULT_REPO, name=DEFAULT_REGEX, timeout=10):
pattern = re.compile(name)
with packages_reader(url, timeout) as lines:
for pkg in parse_packages(lines):
if pattern.match(pkg['package']) is not None:
yield DriverEntry(pkg['package'], upstream_version(pkg['version']))
def get_deb_versions(*args, **kwargs):
return list(set(_get_deb_versions(*args, **kwargs)))
def parse_args():
import argparse
def check_positive_float(val):
val = float(val)
if val <= 0:
raise ValueError("Value %s is not valid positive float" %
(repr(val),))
return val
parser = argparse.ArgumentParser(
description="Retrieves info about latest NVIDIA drivers available in "
"Nvidia deb packages repositories",
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("-U", "--url",
default=DEFAULT_REPO,
help="URL for Packages or Packages.gz file")
parser.add_argument("-N", "--name",
default=DEFAULT_REGEX,
help="Package name regexp")
parser.add_argument("-T", "--timeout",
type=check_positive_float,
default=10.,
help="timeout for network operations")
args = parser.parse_args()
return args
def main():
args = parse_args()
drv = get_deb_versions(url=args.url,
name=args.name,
timeout=args.timeout)
if drv is None:
print("NOT FOUND")
sys.exit(3)
if False: #not args.raw:
print("Version: %s" % (drv['DriverAttributes']['Version'],))
print("Beta: %s" % (bool(int(drv['DriverAttributes']['IsBeta'])),))
print("WHQL: %s" % (bool(int(drv['DriverAttributes']['IsWHQL'])),))
print("URL: %s" % (drv['DriverAttributes']['DownloadURLAdmin'],))
else:
json.dump(drv, sys.stdout, indent=4)
sys.stdout.flush()
if __name__ == '__main__':
main()
tools/nv-driver-locator/nv-driver-locator.json.sample
View file @ e56416e4
......@@ -242,15 +242,70 @@
{
"type": "vulkan_beta",
"name": "vulkan beta windows",
"params": {}
},
{
"type": "deb_packages",
"name": "nvidia cuda drivers deb repo for ubuntu 18.04",
"params": {
"os": "Windows"
"url": "https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/Packages.gz",
"pkg_pattern": "^cuda-drivers$"
}
},
{
"type": "vulkan_beta",
"name": "vulkan beta linux",
"type": "deb_packages",
"name": "nvidia cuda drivers deb repo for ubuntu 20.04",
"params": {
"url": "https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/Packages.gz",
"pkg_pattern": "^cuda-drivers$"
}
},
{
"type": "deb_packages",
"name": "ubuntu 18.04 official repos",
"params": {
"url": "http://security.ubuntu.com/ubuntu/dists/bionic/restricted/binary-amd64/Packages.gz",
"pkg_pattern": "^nvidia-driver-\\d+$"
}
},
{
"type": "deb_packages",
"name": "ubuntu 18.04 official repos (updates)",
"params": {
"url": "http://security.ubuntu.com/ubuntu/dists/bionic-updates/restricted/binary-amd64/Packages.gz",
"pkg_pattern": "^nvidia-driver-\\d+$"
}
},
{
"type": "deb_packages",
"name": "ubuntu 18.04 official repos (security)",
"params": {
"url": "http://security.ubuntu.com/ubuntu/dists/bionic-security/restricted/binary-amd64/Packages.gz",
"pkg_pattern": "^nvidia-driver-\\d+$"
}
},
{
"type": "deb_packages",
"name": "ubuntu 20.04 official repos",
"params": {
"url": "http://security.ubuntu.com/ubuntu/dists/focal/restricted/binary-amd64/Packages.gz",
"pkg_pattern": "^nvidia-driver-\\d+$"
}
},
{
"type": "deb_packages",
"name": "ubuntu 20.04 official repos (updates)",
"params": {
"url": "http://security.ubuntu.com/ubuntu/dists/focal-updates/restricted/binary-amd64/Packages.gz",
"pkg_pattern": "^nvidia-driver-\\d+$"
}
},
{
"type": "deb_packages",
"name": "ubuntu 20.04 official repos (security)",
"params": {
"os": "Linux"
"url": "http://security.ubuntu.com/ubuntu/dists/focal-security/restricted/binary-amd64/Packages.gz",
"pkg_pattern": "^nvidia-driver-\\d+$"
}
}
],
......
tools/nv-driver-locator/nv-driver-locator.py
View file @ e56416e4
......@@ -150,7 +150,7 @@ class CommandNotifier(BaseNotifier):
class BaseChannel(ABC):
@abstractmethod
def get_latest_driver(self):
def get_latest_drivers(self):
pass
......@@ -175,18 +175,20 @@ class GFEClientChannel(BaseChannel):
self._crd = crd
self._timeout = timeout
gfe_get_driver = importlib.import_module('gfe_get_driver')
self._get_latest_driver = gfe_get_driver.get_latest_geforce_driver
def get_latest_driver(self):
res = self._get_latest_driver(notebook=self._notebook,
x86_64=self._x86_64,
os_version=self._os_version,
os_build=self._os_build,
language=self._language,
beta=self._beta,
dch=self._dch,
crd=self._crd,
timeout=self._timeout)
self._get_latest_drivers = gfe_get_driver.get_latest_geforce_driver
def get_latest_drivers(self):
res = self._get_latest_drivers(notebook=self._notebook,
x86_64=self._x86_64,
os_version=self._os_version,
os_build=self._os_build,
language=self._language,
beta=self._beta,
dch=self._dch,
crd=self._crd,
timeout=self._timeout)
if res is None:
return
res.update({
'ChannelAttributes': {
'Name': self.name,
......@@ -201,7 +203,7 @@ class GFEClientChannel(BaseChannel):
'Mobile': self._notebook,
}
})
return res
yield res
class NvidiaDownloadsChannel(BaseChannel):
......@@ -224,7 +226,7 @@ class NvidiaDownloadsChannel(BaseChannel):
self._cuda_ver = gnd.CUDAToolkitVersion[cuda_ver]
self._timeout = timeout
def get_latest_driver(self):
def get_latest_drivers(self):
latest = self._gnd.get_drivers(os=self._os,
product=self._product,
certlevel=self._certlevel,
......@@ -233,7 +235,7 @@ class NvidiaDownloadsChannel(BaseChannel):
cuda_ver=self._cuda_ver,
timeout=self._timeout)
if not latest:
return None
return
res = {
'DriverAttributes': {
'Version': latest['version'],
......@@ -253,7 +255,7 @@ class NvidiaDownloadsChannel(BaseChannel):
}
if 'download_url' in latest:
res['DriverAttributes']['DownloadURL'] = latest['download_url']
return res
yield res
class CudaToolkitDownloadsChannel(BaseChannel):
......@@ -264,45 +266,81 @@ class CudaToolkitDownloadsChannel(BaseChannel):
self._gcd = gcd
self._timeout = timeout
def get_latest_driver(self):
def get_latest_drivers(self):
latest = self._gcd.get_latest_cuda_tk(timeout=self._timeout)
if not latest:
return None
return {
return
yield {
'DriverAttributes': {
'Version': '???',
'Name': latest,
'NameLocalized': latest,
},
'ChannelAttributes': {
'Name': self.name,
'Type': self.__class__.__name__,
}
}
@functools.lru_cache(maxsize=0)
def vulkan_downloads(*, timeout=10):
gvd = importlib.import_module('get_vulkan_downloads')
return gvd.get_drivers(timeout=timeout)
class VulkanBetaDownloadsChannel(BaseChannel):
def __init__(self, name, *,
os="Linux",
timeout=10):
self.name = name
self._os = os
self._timeout = timeout
self._gvd = importlib.import_module('get_vulkan_downloads')
def get_latest_driver(self):
drivers = vulkan_downloads(timeout=self._timeout)
def get_latest_drivers(self):
drivers = self._gvd.get_drivers(timeout=self._timeout)
if drivers is None:
return
for drv in drivers:
if drv["os"] == self._os:
return {
'DriverAttributes': {
'Version': drv['version'],
'Name': drv['name'],
'NameLocalized': drv['name'],
}
yield {
'DriverAttributes': {
'Version': drv['version'],
'Name': drv['name'],
'NameLocalized': drv['name'],
},
'ChannelAttributes': {
'Name': self.name,
'Type': self.__class__.__name__,
'OS': drv['os'],
}
else:
return None
}
class DebRepoChannel(BaseChannel):
def __init__(self, name, *,
url,
pkg_pattern,
driver_name="Linux x64 (AMD64/EM64T) Display Driver",
timeout=10):
self.name = name
self._gdd = importlib.import_module('get_deb_drivers')
self._url = url
self._pkg_pattern = pkg_pattern
self._driver_name = driver_name
self._timeout = timeout
def get_latest_drivers(self):
drivers = self._gdd.get_deb_versions(url=self._url,
name=self._pkg_pattern,
timeout=self._timeout)
if drivers is None:
return
for drv in drivers:
yield {
'DriverAttributes': {
'Version': drv.version,
'DebPkgName': drv.name,
'Name': self._driver_name,
'NameLocalized': self._driver_name,
},
'ChannelAttributes': {
'Name': self.name,
'Type': self.__class__.__name__,
'OS': 'Linux_64',
'PkgPattern': self._pkg_pattern,
}
}
def parse_args():
parser = argparse.ArgumentParser(
......@@ -332,6 +370,7 @@ class DriverLocator:
'nvidia_downloads': NvidiaDownloadsChannel,
'cuda_downloads': CudaToolkitDownloadsChannel,
'vulkan_beta': VulkanBetaDownloadsChannel,
'deb_packages': DebRepoChannel,
}
channels = []
......@@ -389,26 +428,39 @@ class DriverLocator:
def run(self):
for ch in self._channels:
counter = 0
try:
drv = ch.get_latest_driver()
drivers = ch.get_latest_drivers()
except Exception as e:
self._perror("get_latest_driver() invocation failed for "
self._perror("get_latest_drivers() invocation failed for "
"channel %s. Exception: %s. Continuing..." %
(repr(ch.name), str(e)))
continue
if drv is None:
self._perror("Driver not found for channel %s" %
(repr(ch.name),))
continue
try:
key = self._hasher.hash_object(drv)
# Fetch
for drv in drivers:
counter += 1
# Hash
try:
key = self._hasher.hash_object(drv)
except Exception as e:
self._perror("Key evaluation failed for channel %s. "
"Exception: %s" % (repr(name), str(e)))
continue
# Notify
if not self._db.check_key(key):
if self._notify_all(drv):
self._db.set_key(key, drv)
except Exception as e:
self._perror("Key evaluation failed for channel %s. "
"Exception: %s" % (repr(name), str(e)))
self._perror("channel %s enumeration terminated with exception: %s" %
(repr(name), str(e)))
continue
if not self._db.check_key(key):
if self._notify_all(drv):
self._db.set_key(key, drv)
if not counter:
self._perror("Drivers not found for channel %s" %
(repr(ch.name),))
return self._ret_code
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment