From 80de7ed5cb926d308a5bb7c645bd17e77b482428 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Wed, 27 Jul 2022 08:27:28 -0700 Subject: [PATCH 001/116] version bump. locale info changes. add ip-address parser --- CHANGELOG | 4 ++++ jc/cli.py | 4 ++-- jc/lib.py | 3 ++- setup.py | 2 +- templates/readme_template | 9 +++++---- 5 files changed, 14 insertions(+), 8 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 916d5ff4b..81818ca83 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,5 +1,9 @@ jc changelog +xxxxxxxx v1.20.5 +- Add IP Address string parser +- Change LANG=C to LC_ALL=C in locale instructions + 20220723 v1.20.4 - Fix URL string parser path list for URLs ending in a forward slash diff --git a/jc/cli.py b/jc/cli.py index 19f52b313..7264b50f3 100644 --- a/jc/cli.py +++ b/jc/cli.py @@ -638,7 +638,7 @@ def main(): utils.error_message([ f'Parser issue with {parser_name}:', f'{e.__class__.__name__}: {e}', - 'If this is the correct parser, try setting the locale to C (LANG=C).', + 'If this is the correct parser, try setting the locale to C (LC_ALL=C).', f'For details use the -d or -dd option. Use "jc -h --{parser_name}" for help.' ]) sys.exit(combined_exit_code(magic_exit_code, JC_ERROR_EXIT)) @@ -654,7 +654,7 @@ def main(): utils.error_message([ f'{parser_name} parser could not parse the input data.', f'{streaming_msg}', - 'If this is the correct parser, try setting the locale to C (LANG=C).', + 'If this is the correct parser, try setting the locale to C (LC_ALL=C).', f'For details use the -d or -dd option. Use "jc -h --{parser_name}" for help.' ]) sys.exit(combined_exit_code(magic_exit_code, JC_ERROR_EXIT)) diff --git a/jc/lib.py b/jc/lib.py index 9db1d5aa1..e554fbd95 100644 --- a/jc/lib.py +++ b/jc/lib.py @@ -6,7 +6,7 @@ from typing import Dict, List, Iterable, Union, Iterator from jc import appdirs -__version__ = '1.20.4' +__version__ = '1.20.5' parsers = [ 'acpi', @@ -50,6 +50,7 @@ 'ini', 'iostat', 'iostat-s', + 'ip-address', 'iptables', 'iso-datetime', 'iw-scan', diff --git a/setup.py b/setup.py index 6a12598c8..5f39bc3cb 100755 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setuptools.setup( name='jc', - version='1.20.4', + version='1.20.5', author='Kelly Brazil', author_email='kellyjonbrazil@gmail.com', description='Converts the output of popular command-line tools and file-types to JSON.', diff --git a/templates/readme_template b/templates/readme_template index 8f5c6e96c..1db104f2c 100644 --- a/templates/readme_template +++ b/templates/readme_template @@ -322,15 +322,16 @@ Local plugins may override default parsers. #### Locale -For best results set the `LANG` locale environment variable to `C` or -`en_US.UTF-8`. For example, either by setting directly on the command-line: +For best results set the locale environment variables to `C` or +`en_US.UTF-8` by modifying the `LC_ALL` variable: ``` -$ LANG=C date | jc --date +$ LC_ALL=C date | jc --date ``` -or by exporting to the environment before running commands: +You can also set the locale variables individually: ``` $ export LANG=C +$ export LC_NUMERIC=C ``` On some older systems UTF-8 output will be downgraded to ASCII with `\\u` From 9fcf1eb9cb6172fba301412c48ddf73c3409f5d7 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Wed, 27 Jul 2022 10:42:35 -0700 Subject: [PATCH 002/116] initial ip-address parser --- jc/parsers/ip_address.py | 123 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 jc/parsers/ip_address.py diff --git a/jc/parsers/ip_address.py b/jc/parsers/ip_address.py new file mode 100644 index 000000000..7d16d76b0 --- /dev/null +++ b/jc/parsers/ip_address.py @@ -0,0 +1,123 @@ +"""jc - JSON Convert IP Address string parser + +Usage (cli): + + $ echo '192.168.1.1' | jc --ip-address + +Usage (module): + + import jc + result = jc.parse('ip_address', ip_address_string) + +Schema: + + [ + { + "ip_address": string, + "bar": boolean, + "baz": integer + } + ] + +Examples: + + $ ip_address | jc --ip_address -p + [] + + $ ip_address | jc --ip_address -p -r + [] +""" +from typing import List, Dict +import binascii +import ipaddress +import jc.utils + + +class info(): + """Provides parser metadata (version, author, etc.)""" + version = '1.0' + description = 'IP Address string parser' + author = 'Kelly Brazil' + author_email = 'kellyjonbrazil@gmail.com' + compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd'] + + +__version__ = info.version + + +def _process(proc_data: Dict) -> Dict: + """ + Final processing to conform to the schema. + + Parameters: + + proc_data: (Dictionary) raw structured data to process + + Returns: + + Dictionary. Structured to conform to the schema. + """ + return proc_data + + +def _b2a(byte_string: bytes) -> str: + """Convert a byte string to a colon-delimited hex ascii string""" + # need try/except since seperator was only introduced in python 3.8. + # provides compatibility for python 3.6 and 3.7. + try: + return binascii.hexlify(byte_string, ':').decode('utf-8') + except TypeError: + hex_string = binascii.hexlify(byte_string).decode('utf-8') + colon_seperated = ':'.join(hex_string[i:i+2] for i in range(0, len(hex_string), 2)) + return colon_seperated + + +def parse( + data: str, + raw: bool = False, + quiet: bool = False +) -> Dict: + """ + Main text parsing function + + Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + + Returns: + + Dictionary. Raw or processed structured data. + """ + jc.utils.compatibility(__name__, info.compatible, quiet) + jc.utils.input_type_check(data) + + raw_output: Dict = {} + + if jc.utils.has_data(data): + + interface = ipaddress.ip_interface(data.strip()) + + raw_output = { + 'version': int(interface.version), + 'ip': str(interface.ip), + 'ip_compressed': str(interface.compressed), + 'ip_exploded': str(interface.exploded), + 'ip_hex': _b2a(interface.packed), + 'dns_ptr': str(interface.reverse_pointer), + 'network': str(interface.network).split('/')[0], + 'netmask': str(interface.with_netmask).split('/')[1], + 'cidr_netmask': str(interface.with_prefixlen).split('/')[1], + 'hostmask': str(interface.with_hostmask).split('/')[1], + 'max_prefix_length': int(interface.max_prefixlen), + 'is_multicast': interface.is_multicast, + 'is_private': interface.is_private, + 'is_global': interface.is_global, + 'is_link_local': interface.is_link_local, + 'is_loopback': interface.is_loopback, + 'is_reserved': interface.is_reserved, + 'is_unspecified': interface.is_unspecified + } + + return raw_output if raw else _process(raw_output) From deaf1860fb955f44cf4d4fead64447c64f6cedeb Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Wed, 27 Jul 2022 11:42:11 -0700 Subject: [PATCH 003/116] add fields --- jc/parsers/ip_address.py | 35 +++++++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/jc/parsers/ip_address.py b/jc/parsers/ip_address.py index 7d16d76b0..07387fa32 100644 --- a/jc/parsers/ip_address.py +++ b/jc/parsers/ip_address.py @@ -98,26 +98,49 @@ def parse( if jc.utils.has_data(data): interface = ipaddress.ip_interface(data.strip()) + network_string = str(interface.network).split('/')[0] + network_cidr = str(interface.with_prefixlen).split('/')[1] + network = ipaddress.ip_network(f'{network_string}/{network_cidr}') raw_output = { 'version': int(interface.version), + 'max_prefix_length': int(interface.max_prefixlen), 'ip': str(interface.ip), 'ip_compressed': str(interface.compressed), 'ip_exploded': str(interface.exploded), - 'ip_hex': _b2a(interface.packed), 'dns_ptr': str(interface.reverse_pointer), - 'network': str(interface.network).split('/')[0], - 'netmask': str(interface.with_netmask).split('/')[1], - 'cidr_netmask': str(interface.with_prefixlen).split('/')[1], + 'network': network_string, + 'broadcast': str(network.broadcast_address), 'hostmask': str(interface.with_hostmask).split('/')[1], - 'max_prefix_length': int(interface.max_prefixlen), + 'netmask': str(interface.with_netmask).split('/')[1], + 'cidr_netmask': network_cidr, + 'first_host': 1, # implement + 'last_host': 2, # implement 'is_multicast': interface.is_multicast, 'is_private': interface.is_private, 'is_global': interface.is_global, 'is_link_local': interface.is_link_local, 'is_loopback': interface.is_loopback, 'is_reserved': interface.is_reserved, - 'is_unspecified': interface.is_unspecified + 'is_unspecified': interface.is_unspecified, + 'hex': { + 'ip': _b2a(interface.packed), + 'network': 1, # implement + 'broadcast': 2, # implement + 'hostmask': 3, # implement + 'netmask': 4, # implement + 'first_host': 1, # implement + 'last_host': 2, # implement + }, + 'bin': { + 'ip': 1, # implement + 'network': 1, # implement + 'broadcast': 2, # implement + 'hostmask': 3, # implement + 'netmask': 4, # implement + 'first_host': 1, # implement + 'last_host': 2, # implement + } } return raw_output if raw else _process(raw_output) From 82398aef5ad6fc4cbd5eb3a2c14dacb7d70bbe40 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Wed, 27 Jul 2022 12:40:10 -0700 Subject: [PATCH 004/116] add ranges and fix ptr --- jc/parsers/ip_address.py | 29 +++++-- .../generic/lsusb-device-qualifier.out | 81 +++++++++++++++++++ 2 files changed, 104 insertions(+), 6 deletions(-) create mode 100644 tests/fixtures/generic/lsusb-device-qualifier.out diff --git a/jc/parsers/ip_address.py b/jc/parsers/ip_address.py index 07387fa32..23bcd752c 100644 --- a/jc/parsers/ip_address.py +++ b/jc/parsers/ip_address.py @@ -30,6 +30,7 @@ from typing import List, Dict import binascii import ipaddress +from collections import deque import jc.utils @@ -101,21 +102,37 @@ def parse( network_string = str(interface.network).split('/')[0] network_cidr = str(interface.with_prefixlen).split('/')[1] network = ipaddress.ip_network(f'{network_string}/{network_cidr}') + bare_ip_string = str(interface.ip) + bare_ip = ipaddress.ip_address(bare_ip_string) + ip_ptr = bare_ip.reverse_pointer + + first_host = next(network.hosts()) + + # hack to speed up iterating through large ipv6 subnets + # only do last_host for masks >= /16 (ipv4) or >= /120 (ipv6) + last_host = None + + if any(( + int(interface.version) == 4 and int(network_cidr) >= 16, + int(interface.version) == 6 and int(network_cidr) >= 120, + )): + dd = deque(network.hosts(), maxlen=1) + last_host = str(dd.pop()) raw_output = { 'version': int(interface.version), 'max_prefix_length': int(interface.max_prefixlen), - 'ip': str(interface.ip), + 'ip': bare_ip_string, 'ip_compressed': str(interface.compressed), 'ip_exploded': str(interface.exploded), - 'dns_ptr': str(interface.reverse_pointer), + 'dns_ptr': ip_ptr, 'network': network_string, 'broadcast': str(network.broadcast_address), 'hostmask': str(interface.with_hostmask).split('/')[1], 'netmask': str(interface.with_netmask).split('/')[1], - 'cidr_netmask': network_cidr, - 'first_host': 1, # implement - 'last_host': 2, # implement + 'cidr_netmask': int(network_cidr), + 'first_host': str(first_host), + 'last_host': last_host, # None if netmask is too small 'is_multicast': interface.is_multicast, 'is_private': interface.is_private, 'is_global': interface.is_global, @@ -124,7 +141,7 @@ def parse( 'is_reserved': interface.is_reserved, 'is_unspecified': interface.is_unspecified, 'hex': { - 'ip': _b2a(interface.packed), + 'ip': _b2a(bare_ip.packed), 'network': 1, # implement 'broadcast': 2, # implement 'hostmask': 3, # implement diff --git a/tests/fixtures/generic/lsusb-device-qualifier.out b/tests/fixtures/generic/lsusb-device-qualifier.out new file mode 100644 index 000000000..1234e4a01 --- /dev/null +++ b/tests/fixtures/generic/lsusb-device-qualifier.out @@ -0,0 +1,81 @@ + +Bus 002 Device 002: ID 8087:8000 Intel Corp. Integrated Rate Matching Hub +Device Descriptor: + bLength 18 + bDescriptorType 1 + bcdUSB 2.00 + bDeviceClass 9 Hub + bDeviceSubClass 0 + bDeviceProtocol 1 Single TT + bMaxPacketSize0 64 + idVendor 0x8087 Intel Corp. + idProduct 0x8000 Integrated Rate Matching Hub + bcdDevice 0.04 + iManufacturer 0 + iProduct 0 + iSerial 0 + bNumConfigurations 1 + Configuration Descriptor: + bLength 9 + bDescriptorType 2 + wTotalLength 0x0019 + bNumInterfaces 1 + bConfigurationValue 1 + iConfiguration 0 + bmAttributes 0xe0 + Self Powered + Remote Wakeup + MaxPower 0mA + Interface Descriptor: + bLength 9 + bDescriptorType 4 + bInterfaceNumber 0 + bAlternateSetting 0 + bNumEndpoints 1 + bInterfaceClass 9 Hub + bInterfaceSubClass 0 + bInterfaceProtocol 0 Full speed (or root) hub + iInterface 0 + Endpoint Descriptor: + bLength 7 + bDescriptorType 5 + bEndpointAddress 0x81 EP 1 IN + bmAttributes 3 + Transfer Type Interrupt + Synch Type None + Usage Type Data + wMaxPacketSize 0x0002 1x 2 bytes + bInterval 12 +Hub Descriptor: + bLength 11 + bDescriptorType 41 + nNbrPorts 8 + wHubCharacteristic 0x0009 + Per-port power switching + Per-port overcurrent protection + TT think time 8 FS bits + bPwrOn2PwrGood 0 * 2 milli seconds + bHubContrCurrent 0 milli Ampere + DeviceRemovable 0x00 0x00 + PortPwrCtrlMask 0xff 0xff + Hub Port Status: + Port 1: 0000.0100 power + Port 2: 0000.0100 power + Port 3: 0000.0100 power + Port 4: 0000.0100 power + Port 5: 0000.0100 power + Port 6: 0000.0100 power + Port 7: 0000.0100 power + Port 8: 0000.0100 power +Device Qualifier (for other device speed): + bLength 10 + bDescriptorType 6 + bcdUSB 2.00 + bDeviceClass 9 Hub + bDeviceSubClass 0 + bDeviceProtocol 0 Full speed (or root) hub + bMaxPacketSize0 64 + bNumConfigurations 1 +Device Status: 0x0001 + Self Powered + From d970b435b4437841d758a5660e3b8bcd37472e56 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Wed, 27 Jul 2022 12:49:03 -0700 Subject: [PATCH 005/116] do str/int conversions up front --- jc/parsers/ip_address.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/jc/parsers/ip_address.py b/jc/parsers/ip_address.py index 23bcd752c..fbb6a8b61 100644 --- a/jc/parsers/ip_address.py +++ b/jc/parsers/ip_address.py @@ -27,7 +27,7 @@ $ ip_address | jc --ip_address -p -r [] """ -from typing import List, Dict +from typing import Dict import binascii import ipaddress from collections import deque @@ -100,21 +100,21 @@ def parse( interface = ipaddress.ip_interface(data.strip()) network_string = str(interface.network).split('/')[0] - network_cidr = str(interface.with_prefixlen).split('/')[1] + network_cidr = int(str(interface.with_prefixlen).split('/')[1]) network = ipaddress.ip_network(f'{network_string}/{network_cidr}') bare_ip_string = str(interface.ip) bare_ip = ipaddress.ip_address(bare_ip_string) ip_ptr = bare_ip.reverse_pointer - first_host = next(network.hosts()) + first_host = str(next(network.hosts())) # hack to speed up iterating through large ipv6 subnets # only do last_host for masks >= /16 (ipv4) or >= /120 (ipv6) last_host = None if any(( - int(interface.version) == 4 and int(network_cidr) >= 16, - int(interface.version) == 6 and int(network_cidr) >= 120, + int(interface.version) == 4 and network_cidr >= 16, + int(interface.version) == 6 and network_cidr >= 120, )): dd = deque(network.hosts(), maxlen=1) last_host = str(dd.pop()) @@ -130,8 +130,8 @@ def parse( 'broadcast': str(network.broadcast_address), 'hostmask': str(interface.with_hostmask).split('/')[1], 'netmask': str(interface.with_netmask).split('/')[1], - 'cidr_netmask': int(network_cidr), - 'first_host': str(first_host), + 'cidr_netmask': network_cidr, + 'first_host': first_host, 'last_host': last_host, # None if netmask is too small 'is_multicast': interface.is_multicast, 'is_private': interface.is_private, From 11d2eb35be88e23ef63ee1e4d596f198ce5242df Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Wed, 27 Jul 2022 13:08:35 -0700 Subject: [PATCH 006/116] calculate first and last host instead of using the slower iterable --- jc/parsers/ip_address.py | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/jc/parsers/ip_address.py b/jc/parsers/ip_address.py index fbb6a8b61..8d2714e2d 100644 --- a/jc/parsers/ip_address.py +++ b/jc/parsers/ip_address.py @@ -102,22 +102,14 @@ def parse( network_string = str(interface.network).split('/')[0] network_cidr = int(str(interface.with_prefixlen).split('/')[1]) network = ipaddress.ip_network(f'{network_string}/{network_cidr}') + broadcast_string = str(network.broadcast_address) bare_ip_string = str(interface.ip) bare_ip = ipaddress.ip_address(bare_ip_string) ip_ptr = bare_ip.reverse_pointer - first_host = str(next(network.hosts())) - - # hack to speed up iterating through large ipv6 subnets - # only do last_host for masks >= /16 (ipv4) or >= /120 (ipv6) - last_host = None - - if any(( - int(interface.version) == 4 and network_cidr >= 16, - int(interface.version) == 6 and network_cidr >= 120, - )): - dd = deque(network.hosts(), maxlen=1) - last_host = str(dd.pop()) + # TODO: fix for /30, /31, /32, etc. + first_host = str(ipaddress.ip_address(network_string) + 1) + last_host = str(ipaddress.ip_address(broadcast_string) - 1) raw_output = { 'version': int(interface.version), @@ -127,12 +119,12 @@ def parse( 'ip_exploded': str(interface.exploded), 'dns_ptr': ip_ptr, 'network': network_string, - 'broadcast': str(network.broadcast_address), + 'broadcast': broadcast_string, 'hostmask': str(interface.with_hostmask).split('/')[1], 'netmask': str(interface.with_netmask).split('/')[1], 'cidr_netmask': network_cidr, 'first_host': first_host, - 'last_host': last_host, # None if netmask is too small + 'last_host': last_host, 'is_multicast': interface.is_multicast, 'is_private': interface.is_private, 'is_global': interface.is_global, From fdb629f82b85474c70785b81a4708fee7b7dfcc2 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Wed, 27 Jul 2022 14:27:05 -0700 Subject: [PATCH 007/116] fix host number calculations --- jc/parsers/ip_address.py | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/jc/parsers/ip_address.py b/jc/parsers/ip_address.py index 8d2714e2d..42e74a8e8 100644 --- a/jc/parsers/ip_address.py +++ b/jc/parsers/ip_address.py @@ -30,7 +30,6 @@ from typing import Dict import binascii import ipaddress -from collections import deque import jc.utils @@ -107,9 +106,27 @@ def parse( bare_ip = ipaddress.ip_address(bare_ip_string) ip_ptr = bare_ip.reverse_pointer - # TODO: fix for /30, /31, /32, etc. - first_host = str(ipaddress.ip_address(network_string) + 1) - last_host = str(ipaddress.ip_address(broadcast_string) - 1) + # Find first and last host IPs. Fix for /31, /32, /127, /128 + if any(( + int(interface.version) == 4 and network_cidr == 31, + int(interface.version) == 6 and network_cidr == 127 + )): + first_host = str(ipaddress.ip_address(network_string)) + last_host = str(ipaddress.ip_address(broadcast_string)) + hosts = 2 + + elif any(( + int(interface.version) == 4 and network_cidr == 32, + int(interface.version) == 6 and network_cidr == 128 + )): + first_host = bare_ip_string + last_host = bare_ip_string + hosts = 1 + + else: + first_host = str(ipaddress.ip_address(network_string) + 1) + last_host = str(ipaddress.ip_address(broadcast_string) - 1) + hosts = int(ipaddress.ip_address(broadcast_string) - 1) - int(ipaddress.ip_address(network_string) + 1) + 1 raw_output = { 'version': int(interface.version), @@ -123,6 +140,7 @@ def parse( 'hostmask': str(interface.with_hostmask).split('/')[1], 'netmask': str(interface.with_netmask).split('/')[1], 'cidr_netmask': network_cidr, + 'hosts': hosts, 'first_host': first_host, 'last_host': last_host, 'is_multicast': interface.is_multicast, From 624fba9704b8552d1e55b4039bab201b3699481d Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Wed, 27 Jul 2022 15:36:01 -0700 Subject: [PATCH 008/116] add hex and binary representations --- jc/parsers/ip_address.py | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/jc/parsers/ip_address.py b/jc/parsers/ip_address.py index 42e74a8e8..cc6565a68 100644 --- a/jc/parsers/ip_address.py +++ b/jc/parsers/ip_address.py @@ -102,6 +102,8 @@ def parse( network_cidr = int(str(interface.with_prefixlen).split('/')[1]) network = ipaddress.ip_network(f'{network_string}/{network_cidr}') broadcast_string = str(network.broadcast_address) + hostmask_string = str(interface.with_hostmask).split('/')[1] + netmask_string = str(interface.with_netmask).split('/')[1] bare_ip_string = str(interface.ip) bare_ip = ipaddress.ip_address(bare_ip_string) ip_ptr = bare_ip.reverse_pointer @@ -137,8 +139,8 @@ def parse( 'dns_ptr': ip_ptr, 'network': network_string, 'broadcast': broadcast_string, - 'hostmask': str(interface.with_hostmask).split('/')[1], - 'netmask': str(interface.with_netmask).split('/')[1], + 'hostmask': hostmask_string, + 'netmask': netmask_string, 'cidr_netmask': network_cidr, 'hosts': hosts, 'first_host': first_host, @@ -152,21 +154,21 @@ def parse( 'is_unspecified': interface.is_unspecified, 'hex': { 'ip': _b2a(bare_ip.packed), - 'network': 1, # implement - 'broadcast': 2, # implement - 'hostmask': 3, # implement - 'netmask': 4, # implement - 'first_host': 1, # implement - 'last_host': 2, # implement + 'network': _b2a(ipaddress.ip_address(network_string).packed), + 'broadcast': _b2a(ipaddress.ip_address(broadcast_string).packed), + 'hostmask': _b2a(ipaddress.ip_address(hostmask_string).packed), + 'netmask': _b2a(ipaddress.ip_address(netmask_string).packed), + 'first_host': _b2a(ipaddress.ip_address(first_host).packed), + 'last_host': _b2a(ipaddress.ip_address(last_host).packed) }, 'bin': { - 'ip': 1, # implement - 'network': 1, # implement - 'broadcast': 2, # implement - 'hostmask': 3, # implement - 'netmask': 4, # implement - 'first_host': 1, # implement - 'last_host': 2, # implement + 'ip': format(int(bare_ip), '0>' + str(interface.max_prefixlen) +'b'), + 'network': format(int(ipaddress.ip_address(network_string)), '0>' + str(interface.max_prefixlen) +'b'), + 'broadcast': format(int(ipaddress.ip_address(broadcast_string)), '0>' + str(interface.max_prefixlen) +'b'), + 'hostmask': format(int(ipaddress.ip_address(hostmask_string)), '0>' + str(interface.max_prefixlen) +'b'), + 'netmask': format(int(ipaddress.ip_address(netmask_string)), '0>' + str(interface.max_prefixlen) +'b'), + 'first_host': format(int(ipaddress.ip_address(first_host)), '0>' + str(interface.max_prefixlen) +'b'), + 'last_host': format(int(ipaddress.ip_address(last_host)), '0>' + str(interface.max_prefixlen) +'b'), } } From b4fb2d102cd5679d70ea3f6bd232bc71cb573d68 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Wed, 27 Jul 2022 16:22:32 -0700 Subject: [PATCH 009/116] add ipv6-only fields --- jc/parsers/ip_address.py | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/jc/parsers/ip_address.py b/jc/parsers/ip_address.py index cc6565a68..2715b39a7 100644 --- a/jc/parsers/ip_address.py +++ b/jc/parsers/ip_address.py @@ -108,18 +108,31 @@ def parse( bare_ip = ipaddress.ip_address(bare_ip_string) ip_ptr = bare_ip.reverse_pointer + # fix for ipv6-only attributes + scope_id = None + ipv4_mapped = None + sixtofour = None + teredo_client = None + teredo_server = None + if interface.version == 6: + scope_id = interface.scope_id + ipv4_mapped = str(interface.ipv4_mapped) if interface.ipv4_mapped else None + sixtofour = str(interface.sixtofour) if interface.sixtofour else None + teredo_client = str(interface.teredo[1]) if interface.teredo else None + teredo_server = str(interface.teredo[0]) if interface.teredo else None + # Find first and last host IPs. Fix for /31, /32, /127, /128 if any(( - int(interface.version) == 4 and network_cidr == 31, - int(interface.version) == 6 and network_cidr == 127 + interface.version == 4 and network_cidr == 31, + interface.version == 6 and network_cidr == 127 )): first_host = str(ipaddress.ip_address(network_string)) last_host = str(ipaddress.ip_address(broadcast_string)) hosts = 2 elif any(( - int(interface.version) == 4 and network_cidr == 32, - int(interface.version) == 6 and network_cidr == 128 + interface.version == 4 and network_cidr == 32, + interface.version == 6 and network_cidr == 128 )): first_host = bare_ip_string last_host = bare_ip_string @@ -132,10 +145,15 @@ def parse( raw_output = { 'version': int(interface.version), - 'max_prefix_length': int(interface.max_prefixlen), + 'max_prefix_length': interface.max_prefixlen, 'ip': bare_ip_string, 'ip_compressed': str(interface.compressed), 'ip_exploded': str(interface.exploded), + 'scope_id': scope_id, + 'ipv4_mapped': ipv4_mapped, + 'six_to_four': sixtofour, + 'teredo_client': teredo_client, + 'teredo_server': teredo_server, 'dns_ptr': ip_ptr, 'network': network_string, 'broadcast': broadcast_string, From 726bcc83d2657bd253c01a9a7fed46a39c3e3fb6 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Wed, 27 Jul 2022 17:21:04 -0700 Subject: [PATCH 010/116] add docs --- README.md | 10 +- completions/jc_bash_completion.sh | 2 +- completions/jc_zsh_completion.sh | 3 +- docs/parsers/ip_address.md | 490 ++++++++++++++++++++++++++++++ jc/parsers/ip_address.py | 469 +++++++++++++++++++++++++++- man/jc.1 | 7 +- 6 files changed, 962 insertions(+), 19 deletions(-) create mode 100644 docs/parsers/ip_address.md diff --git a/README.md b/README.md index 07d29f9ab..092f1100e 100644 --- a/README.md +++ b/README.md @@ -189,6 +189,7 @@ option. | ` --ini` | INI file parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/ini) | | ` --iostat` | `iostat` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/iostat) | | ` --iostat-s` | `iostat` command streaming parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/iostat_s) | +| ` --ip-address` | IP Address string parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/ip_address) | | ` --iptables` | `iptables` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/iptables) | | ` --iso-datetime` | ISO 8601 Datetime string parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/iso_datetime) | | ` --iw-scan` | `iw dev [device] scan` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/iw_scan) | @@ -432,15 +433,16 @@ Local plugins may override default parsers. #### Locale -For best results set the `LANG` locale environment variable to `C` or -`en_US.UTF-8`. For example, either by setting directly on the command-line: +For best results set the locale environment variables to `C` or +`en_US.UTF-8` by modifying the `LC_ALL` variable: ``` -$ LANG=C date | jc --date +$ LC_ALL=C date | jc --date ``` -or by exporting to the environment before running commands: +You can also set the locale variables individually: ``` $ export LANG=C +$ export LC_NUMERIC=C ``` On some older systems UTF-8 output will be downgraded to ASCII with `\\u` diff --git a/completions/jc_bash_completion.sh b/completions/jc_bash_completion.sh index 6df9299da..cac8256d0 100644 --- a/completions/jc_bash_completion.sh +++ b/completions/jc_bash_completion.sh @@ -4,7 +4,7 @@ _jc() jc_about_options jc_about_mod_options jc_help_options jc_special_options jc_commands=(acpi airport arp blkid chage cksum crontab date df dig dmidecode dpkg du env file finger free git gpg hciconfig id ifconfig iostat iptables iw jobs last lastb ls lsblk lsmod lsof lsusb md5 md5sum mount mpstat netstat nmcli ntpq pidstat ping ping6 pip pip3 postconf printenv ps route rpm rsync sfdisk sha1sum sha224sum sha256sum sha384sum sha512sum shasum ss stat sum sysctl systemctl systeminfo timedatectl top tracepath tracepath6 traceroute traceroute6 ufw uname update-alternatives upower uptime vdir vmstat w wc who xrandr zipinfo) - jc_parsers=(--acpi --airport --airport-s --arp --asciitable --asciitable-m --blkid --chage --cksum --crontab --crontab-u --csv --csv-s --date --df --dig --dir --dmidecode --dpkg-l --du --email-address --env --file --finger --free --fstab --git-log --git-log-s --gpg --group --gshadow --hash --hashsum --hciconfig --history --hosts --id --ifconfig --ini --iostat --iostat-s --iptables --iso-datetime --iw-scan --jar-manifest --jobs --jwt --kv --last --ls --ls-s --lsblk --lsmod --lsof --lsusb --m3u --mount --mpstat --mpstat-s --netstat --nmcli --ntpq --passwd --pidstat --pidstat-s --ping --ping-s --pip-list --pip-show --postconf --ps --route --rpm-qi --rsync --rsync-s --sfdisk --shadow --ss --stat --stat-s --sysctl --systemctl --systemctl-lj --systemctl-ls --systemctl-luf --systeminfo --time --timedatectl --timestamp --top --top-s --tracepath --traceroute --ufw --ufw-appinfo --uname --update-alt-gs --update-alt-q --upower --uptime --url --vmstat --vmstat-s --w --wc --who --x509-cert --xml --xrandr --yaml --zipinfo) + jc_parsers=(--acpi --airport --airport-s --arp --asciitable --asciitable-m --blkid --chage --cksum --crontab --crontab-u --csv --csv-s --date --df --dig --dir --dmidecode --dpkg-l --du --email-address --env --file --finger --free --fstab --git-log --git-log-s --gpg --group --gshadow --hash --hashsum --hciconfig --history --hosts --id --ifconfig --ini --iostat --iostat-s --ip-address --iptables --iso-datetime --iw-scan --jar-manifest --jobs --jwt --kv --last --ls --ls-s --lsblk --lsmod --lsof --lsusb --m3u --mount --mpstat --mpstat-s --netstat --nmcli --ntpq --passwd --pidstat --pidstat-s --ping --ping-s --pip-list --pip-show --postconf --ps --route --rpm-qi --rsync --rsync-s --sfdisk --shadow --ss --stat --stat-s --sysctl --systemctl --systemctl-lj --systemctl-ls --systemctl-luf --systeminfo --time --timedatectl --timestamp --top --top-s --tracepath --traceroute --ufw --ufw-appinfo --uname --update-alt-gs --update-alt-q --upower --uptime --url --vmstat --vmstat-s --w --wc --who --x509-cert --xml --xrandr --yaml --zipinfo) jc_options=(--force-color -C --debug -d --monochrome -m --pretty -p --quiet -q --raw -r --unbuffer -u --yaml-out -y) jc_about_options=(--about -a) jc_about_mod_options=(--pretty -p --yaml-out -y --monochrome -m --force-color -C) diff --git a/completions/jc_zsh_completion.sh b/completions/jc_zsh_completion.sh index 93f270056..c65c4b905 100644 --- a/completions/jc_zsh_completion.sh +++ b/completions/jc_zsh_completion.sh @@ -94,7 +94,7 @@ _jc() { 'xrandr:run "xrandr" command with magic syntax.' 'zipinfo:run "zipinfo" command with magic syntax.' ) - jc_parsers=(--acpi --airport --airport-s --arp --asciitable --asciitable-m --blkid --chage --cksum --crontab --crontab-u --csv --csv-s --date --df --dig --dir --dmidecode --dpkg-l --du --email-address --env --file --finger --free --fstab --git-log --git-log-s --gpg --group --gshadow --hash --hashsum --hciconfig --history --hosts --id --ifconfig --ini --iostat --iostat-s --iptables --iso-datetime --iw-scan --jar-manifest --jobs --jwt --kv --last --ls --ls-s --lsblk --lsmod --lsof --lsusb --m3u --mount --mpstat --mpstat-s --netstat --nmcli --ntpq --passwd --pidstat --pidstat-s --ping --ping-s --pip-list --pip-show --postconf --ps --route --rpm-qi --rsync --rsync-s --sfdisk --shadow --ss --stat --stat-s --sysctl --systemctl --systemctl-lj --systemctl-ls --systemctl-luf --systeminfo --time --timedatectl --timestamp --top --top-s --tracepath --traceroute --ufw --ufw-appinfo --uname --update-alt-gs --update-alt-q --upower --uptime --url --vmstat --vmstat-s --w --wc --who --x509-cert --xml --xrandr --yaml --zipinfo) + jc_parsers=(--acpi --airport --airport-s --arp --asciitable --asciitable-m --blkid --chage --cksum --crontab --crontab-u --csv --csv-s --date --df --dig --dir --dmidecode --dpkg-l --du --email-address --env --file --finger --free --fstab --git-log --git-log-s --gpg --group --gshadow --hash --hashsum --hciconfig --history --hosts --id --ifconfig --ini --iostat --iostat-s --ip-address --iptables --iso-datetime --iw-scan --jar-manifest --jobs --jwt --kv --last --ls --ls-s --lsblk --lsmod --lsof --lsusb --m3u --mount --mpstat --mpstat-s --netstat --nmcli --ntpq --passwd --pidstat --pidstat-s --ping --ping-s --pip-list --pip-show --postconf --ps --route --rpm-qi --rsync --rsync-s --sfdisk --shadow --ss --stat --stat-s --sysctl --systemctl --systemctl-lj --systemctl-ls --systemctl-luf --systeminfo --time --timedatectl --timestamp --top --top-s --tracepath --traceroute --ufw --ufw-appinfo --uname --update-alt-gs --update-alt-q --upower --uptime --url --vmstat --vmstat-s --w --wc --who --x509-cert --xml --xrandr --yaml --zipinfo) jc_parsers_describe=( '--acpi:`acpi` command parser' '--airport:`airport -I` command parser' @@ -137,6 +137,7 @@ _jc() { '--ini:INI file parser' '--iostat:`iostat` command parser' '--iostat-s:`iostat` command streaming parser' + '--ip-address:IP Address string parser' '--iptables:`iptables` command parser' '--iso-datetime:ISO 8601 Datetime string parser' '--iw-scan:`iw dev [device] scan` command parser' diff --git a/docs/parsers/ip_address.md b/docs/parsers/ip_address.md new file mode 100644 index 000000000..e98bda7a6 --- /dev/null +++ b/docs/parsers/ip_address.md @@ -0,0 +1,490 @@ +[Home](https://kellyjonbrazil.github.io/jc/) + + +# jc.parsers.ip\_address + +jc - JSON Convert IP Address string parser + +Accepts standard and integer IP address notation for both IPv4 and IPv6 +addresses. CIDR subnet mask and Scope ID is also allowed for standard +notation. See examples below. + +Usage (cli): + + $ echo '192.168.1.1' | jc --ip-address + +Usage (module): + + import jc + result = jc.parse('ip_address', ip_address_string) + +Schema: + + { + "version": integer, + "max_prefix_length": integer, + "ip": string, + "ip_compressed": string, + "ip_exploded": string, + "scope_id": string/null, + "ipv4_mapped": string/null, + "six_to_four": string/null, + "teredo_client": string/null, + "teredo_server": string/null, + "dns_ptr": string, + "network": string, + "broadcast": string, + "hostmask": string, + "netmask": string, + "cidr_netmask": integer, + "hosts": integer, + "first_host": string, + "last_host": string, + "is_multicast": boolean, + "is_private": boolean, + "is_global": boolean, + "is_link_local": boolean, + "is_loopback": boolean, + "is_reserved": boolean, + "is_unspecified": boolean, + "int": { + "ip": integer, + "network": integer, + "broadcast": integer, + "first_host": integer, + "last_host": integer + }, + "hex": { + "ip": string, + "network": string, + "broadcast": string, + "hostmask": string, + "netmask": string, + "first_host": string, + "last_host": string + }, + "bin": { + "ip": string, + "network": string, + "broadcast": string, + "hostmask": string, + "netmask": string, + "first_host": string, + "last_host": string + } + } + +Examples: + + $ echo 192.168.2.10/24 | jc --ip-address -p + { + "version": 4, + "max_prefix_length": 32, + "ip": "192.168.2.10", + "ip_compressed": "192.168.2.10/24", + "ip_exploded": "192.168.2.10/24", + "scope_id": null, + "ipv4_mapped": null, + "six_to_four": null, + "teredo_client": null, + "teredo_server": null, + "dns_ptr": "10.2.168.192.in-addr.arpa", + "network": "192.168.2.0", + "broadcast": "192.168.2.255", + "hostmask": "0.0.0.255", + "netmask": "255.255.255.0", + "cidr_netmask": 24, + "hosts": 254, + "first_host": "192.168.2.1", + "last_host": "192.168.2.254", + "is_multicast": false, + "is_private": true, + "is_global": false, + "is_link_local": false, + "is_loopback": false, + "is_reserved": false, + "is_unspecified": false, + "int": { + "ip": 3232236042, + "network": 3232236032, + "broadcast": 3232236287, + "first_host": 3232236033, + "last_host": 3232236286 + }, + "hex": { + "ip": "c0:a8:02:0a", + "network": "c0:a8:02:00", + "broadcast": "c0:a8:02:ff", + "hostmask": "00:00:00:ff", + "netmask": "ff:ff:ff:00", + "first_host": "c0:a8:02:01", + "last_host": "c0:a8:02:fe" + }, + "bin": { + "ip": "11000000101010000000001000001010", + "network": "11000000101010000000001000000000", + "broadcast": "11000000101010000000001011111111", + "hostmask": "00000000000000000000000011111111", + "netmask": "11111111111111111111111100000000", + "first_host": "11000000101010000000001000000001", + "last_host": "11000000101010000000001011111110" + } + } + + $ echo 3232236042 | jc --ip-address -p + { + "version": 4, + "max_prefix_length": 32, + "ip": "192.168.2.10", + "ip_compressed": "192.168.2.10/32", + "ip_exploded": "192.168.2.10/32", + "scope_id": null, + "ipv4_mapped": null, + "six_to_four": null, + "teredo_client": null, + "teredo_server": null, + "dns_ptr": "10.2.168.192.in-addr.arpa", + "network": "192.168.2.10", + "broadcast": "192.168.2.10", + "hostmask": "0.0.0.0", + "netmask": "255.255.255.255", + "cidr_netmask": 32, + "hosts": 1, + "first_host": "192.168.2.10", + "last_host": "192.168.2.10", + "is_multicast": false, + "is_private": true, + "is_global": false, + "is_link_local": false, + "is_loopback": false, + "is_reserved": false, + "is_unspecified": false, + "int": { + "ip": 3232236042, + "network": 3232236042, + "broadcast": 3232236042, + "first_host": 3232236042, + "last_host": 3232236042 + }, + "hex": { + "ip": "c0:a8:02:0a", + "network": "c0:a8:02:0a", + "broadcast": "c0:a8:02:0a", + "hostmask": "00:00:00:00", + "netmask": "ff:ff:ff:ff", + "first_host": "c0:a8:02:0a", + "last_host": "c0:a8:02:0a" + }, + "bin": { + "ip": "11000000101010000000001000001010", + "network": "11000000101010000000001000001010", + "broadcast": "11000000101010000000001000001010", + "hostmask": "00000000000000000000000000000000", + "netmask": "11111111111111111111111111111111", + "first_host": "11000000101010000000001000001010", + "last_host": "11000000101010000000001000001010" + } + } + + $ echo 127:0:de::1%128/96 | jc --ip-address -p + { + "version": 6, + "max_prefix_length": 128, + "ip": "127:0:de::1", + "ip_compressed": "127:0:de::1%128/96", + "ip_exploded": "0127:0000:00de:0000:0000:0000:0000:0001/96", + "scope_id": "128", + "ipv4_mapped": null, + "six_to_four": null, + "teredo_client": null, + "teredo_server": null, + "dns_ptr": "1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.....0.7.2.1.0.ip6.arpa", + "network": "127:0:de::", + "broadcast": "127:0:de::ffff:ffff", + "hostmask": "::ffff:ffff", + "netmask": "ffff:ffff:ffff:ffff:ffff:ffff::", + "cidr_netmask": 96, + "hosts": 4294967294, + "first_host": "127:0:de::1", + "last_host": "127:0:de::ffff:fffe", + "is_multicast": false, + "is_private": false, + "is_global": true, + "is_link_local": false, + "is_loopback": false, + "is_reserved": true, + "is_unspecified": false, + "int": { + "ip": 1531727573536155682370944093904699393, + "network": 1531727573536155682370944093904699392, + "broadcast": 1531727573536155682370944098199666687, + "first_host": 1531727573536155682370944093904699393, + "last_host": 1531727573536155682370944098199666686 + }, + "hex": { + "ip": "01:27:00:00:00:de:00:00:00:00:00:00:00:00:00:01", + "network": "01:27:00:00:00:de:00:00:00:00:00:00:00:00:00:00", + "broadcast": "01:27:00:00:00:de:00:00:00:00:00:00:ff:ff:ff:ff", + "hostmask": "00:00:00:00:00:00:00:00:00:00:00:00:ff:ff:ff:ff", + "netmask": "ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:00:00:00:00", + "first_host": "01:27:00:00:00:de:00:00:00:00:00:00:00:00:00:01", + "last_host": "01:27:00:00:00:de:00:00:00:00:00:00:ff:ff:ff:fe" + }, + "bin": { + "ip": "000000010010011100000000000000000000000011011110000000...", + "network": "0000000100100111000000000000000000000000110111100...", + "broadcast": "00000001001001110000000000000000000000001101111...", + "hostmask": "000000000000000000000000000000000000000000000000...", + "netmask": "1111111111111111111111111111111111111111111111111...", + "first_host": "0000000100100111000000000000000000000000110111...", + "last_host": "00000001001001110000000000000000000000001101111..." + } + } + + $ echo 1531727573536155682370944093904699393 | jc --ip-address -p + { + "version": 6, + "max_prefix_length": 128, + "ip": "127:0:de::1", + "ip_compressed": "127:0:de::1/128", + "ip_exploded": "0127:0000:00de:0000:0000:0000:0000:0001/128", + "scope_id": null, + "ipv4_mapped": null, + "six_to_four": null, + "teredo_client": null, + "teredo_server": null, + "dns_ptr": "1.0.0.0.0.0.0....0.0.0.e.d.0.0.0.0.0.0.7.2.1.0.ip6.arpa", + "network": "127:0:de::1", + "broadcast": "127:0:de::1", + "hostmask": "::", + "netmask": "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", + "cidr_netmask": 128, + "hosts": 1, + "first_host": "127:0:de::1", + "last_host": "127:0:de::1", + "is_multicast": false, + "is_private": false, + "is_global": true, + "is_link_local": false, + "is_loopback": false, + "is_reserved": true, + "is_unspecified": false, + "int": { + "ip": 1531727573536155682370944093904699393, + "network": 1531727573536155682370944093904699393, + "broadcast": 1531727573536155682370944093904699393, + "first_host": 1531727573536155682370944093904699393, + "last_host": 1531727573536155682370944093904699393 + }, + "hex": { + "ip": "01:27:00:00:00:de:00:00:00:00:00:00:00:00:00:01", + "network": "01:27:00:00:00:de:00:00:00:00:00:00:00:00:00:01", + "broadcast": "01:27:00:00:00:de:00:00:00:00:00:00:00:00:00:01", + "hostmask": "00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00", + "netmask": "ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff", + "first_host": "01:27:00:00:00:de:00:00:00:00:00:00:00:00:00:01", + "last_host": "01:27:00:00:00:de:00:00:00:00:00:00:00:00:00:01" + }, + "bin": { + "ip": "0000000100100111000000000000000000000000110111100000000...", + "network": "00000001001001110000000000000000000000001101111000...", + "broadcast": "000000010010011100000000000000000000000011011110...", + "hostmask": "0000000000000000000000000000000000000000000000000...", + "netmask": "11111111111111111111111111111111111111111111111111...", + "first_host": "00000001001001110000000000000000000000001101111...", + "last_host": "000000010010011100000000000000000000000011011110..." + } + } + + # IPv4 Mapped Address + $ echo ::FFFF:192.168.1.35 | jc --ip-address -p + { + "version": 6, + "max_prefix_length": 128, + "ip": "::ffff:c0a8:123", + "ip_compressed": "::ffff:c0a8:123/128", + "ip_exploded": "0000:0000:0000:0000:0000:ffff:c0a8:0123/128", + "scope_id": null, + "ipv4_mapped": "192.168.1.35", + "six_to_four": null, + "teredo_client": null, + "teredo_server": null, + "dns_ptr": "3.2.1.0.8.a.0.c.f.f.f.f.0.0.0....0.0.0.0.0.0.0.ip6.arpa", + "network": "::ffff:c0a8:123", + "broadcast": "::ffff:c0a8:123", + "hostmask": "::", + "netmask": "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", + "cidr_netmask": 128, + "hosts": 1, + "first_host": "::ffff:c0a8:123", + "last_host": "::ffff:c0a8:123", + "is_multicast": false, + "is_private": true, + "is_global": false, + "is_link_local": false, + "is_loopback": false, + "is_reserved": true, + "is_unspecified": false, + "int": { + "ip": 281473913979171, + "network": 281473913979171, + "broadcast": 281473913979171, + "first_host": 281473913979171, + "last_host": 281473913979171 + }, + "hex": { + "ip": "00:00:00:00:00:00:00:00:00:00:ff:ff:c0:a8:01:23", + "network": "00:00:00:00:00:00:00:00:00:00:ff:ff:c0:a8:01:23", + "broadcast": "00:00:00:00:00:00:00:00:00:00:ff:ff:c0:a8:01:23", + "hostmask": "00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00", + "netmask": "ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff", + "first_host": "00:00:00:00:00:00:00:00:00:00:ff:ff:c0:a8:01:23", + "last_host": "00:00:00:00:00:00:00:00:00:00:ff:ff:c0:a8:01:23" + }, + "bin": { + "ip": "0000000000000000000000000000000000000000000000000000000...", + "network": "00000000000000000000000000000000000000000000000000...", + "broadcast": "000000000000000000000000000000000000000000000000...", + "hostmask": "0000000000000000000000000000000000000000000000000...", + "netmask": "11111111111111111111111111111111111111111111111111...", + "first_host": "00000000000000000000000000000000000000000000000...", + "last_host": "000000000000000000000000000000000000000000000000..." + } + } + + # 6to4 Address + $ echo 2002:c000:204::/48 | jc --ip-address -p + { + "version": 6, + "max_prefix_length": 128, + "ip": "2002:c000:204::", + "ip_compressed": "2002:c000:204::/48", + "ip_exploded": "2002:c000:0204:0000:0000:0000:0000:0000/48", + "scope_id": null, + "ipv4_mapped": null, + "six_to_four": "192.0.2.4", + "teredo_client": null, + "teredo_server": null, + "dns_ptr": "0.0.0.0.0.0.0.0......0.4.0.2.0.0.0.0.c.2.0.0.2.ip6.arpa", + "network": "2002:c000:204::", + "broadcast": "2002:c000:204:ffff:ffff:ffff:ffff:ffff", + "hostmask": "::ffff:ffff:ffff:ffff:ffff", + "netmask": "ffff:ffff:ffff::", + "cidr_netmask": 48, + "hosts": 1208925819614629174706174, + "first_host": "2002:c000:204::1", + "last_host": "2002:c000:204:ffff:ffff:ffff:ffff:fffe", + "is_multicast": false, + "is_private": false, + "is_global": true, + "is_link_local": false, + "is_loopback": false, + "is_reserved": false, + "is_unspecified": false, + "int": { + "ip": 42549574682102084431821433448024768512, + "network": 42549574682102084431821433448024768512, + "broadcast": 42549574682103293357641048077199474687, + "first_host": 42549574682102084431821433448024768513, + "last_host": 42549574682103293357641048077199474686 + }, + "hex": { + "ip": "20:02:c0:00:02:04:00:00:00:00:00:00:00:00:00:00", + "network": "20:02:c0:00:02:04:00:00:00:00:00:00:00:00:00:00", + "broadcast": "20:02:c0:00:02:04:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff", + "hostmask": "00:00:00:00:00:00:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff", + "netmask": "ff:ff:ff:ff:ff:ff:00:00:00:00:00:00:00:00:00:00", + "first_host": "20:02:c0:00:02:04:00:00:00:00:00:00:00:00:00:01", + "last_host": "20:02:c0:00:02:04:ff:ff:ff:ff:ff:ff:ff:ff:ff:fe" + }, + "bin": { + "ip": "0010000000000010110000000000000000000010000001000000000...", + "network": "00100000000000101100000000000000000000100000010000...", + "broadcast": "001000000000001011000000000000000000001000000100...", + "hostmask": "0000000000000000000000000000000000000000000000001...", + "netmask": "11111111111111111111111111111111111111111111111100...", + "first_host": "00100000000000101100000000000000000000100000010...", + "last_host": "001000000000001011000000000000000000001000000100..." + } + } + + # Teredo Address + % echo 2001:0000:4136:e378:8000:63bf:3fff:fdd2 | jc --ip-address -p + { + "version": 6, + "max_prefix_length": 128, + "ip": "2001:0:4136:e378:8000:63bf:3fff:fdd2", + "ip_compressed": "2001:0:4136:e378:8000:63bf:3fff:fdd2/128", + "ip_exploded": "2001:0000:4136:e378:8000:63bf:3fff:fdd2/128", + "scope_id": null, + "ipv4_mapped": null, + "six_to_four": null, + "teredo_client": "192.0.2.45", + "teredo_server": "65.54.227.120", + "dns_ptr": "2.d.d.f.f.f.f.3.f.b.3.6.0.0.0....0.0.0.1.0.0.2.ip6.arpa", + "network": "2001:0:4136:e378:8000:63bf:3fff:fdd2", + "broadcast": "2001:0:4136:e378:8000:63bf:3fff:fdd2", + "hostmask": "::", + "netmask": "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", + "cidr_netmask": 128, + "hosts": 1, + "first_host": "2001:0:4136:e378:8000:63bf:3fff:fdd2", + "last_host": "2001:0:4136:e378:8000:63bf:3fff:fdd2", + "is_multicast": false, + "is_private": true, + "is_global": false, + "is_link_local": false, + "is_loopback": false, + "is_reserved": false, + "is_unspecified": false, + "int": { + "ip": 42540488182158724593221357832373272018, + "network": 42540488182158724593221357832373272018, + "broadcast": 42540488182158724593221357832373272018, + "first_host": 42540488182158724593221357832373272018, + "last_host": 42540488182158724593221357832373272018 + }, + "hex": { + "ip": "20:01:00:00:41:36:e3:78:80:00:63:bf:3f:ff:fd:d2", + "network": "20:01:00:00:41:36:e3:78:80:00:63:bf:3f:ff:fd:d2", + "broadcast": "20:01:00:00:41:36:e3:78:80:00:63:bf:3f:ff:fd:d2", + "hostmask": "00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00", + "netmask": "ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff", + "first_host": "20:01:00:00:41:36:e3:78:80:00:63:bf:3f:ff:fd:d2", + "last_host": "20:01:00:00:41:36:e3:78:80:00:63:bf:3f:ff:fd:d2" + }, + "bin": { + "ip": "001000000000000100000000000000000100000100110110111000...", + "network": "0010000000000001000000000000000001000001001101101...", + "broadcast": "00100000000000010000000000000000010000010011011...", + "hostmask": "000000000000000000000000000000000000000000000000...", + "netmask": "1111111111111111111111111111111111111111111111111...", + "first_host": "0010000000000001000000000000000001000001001101...", + "last_host": "00100000000000010000000000000000010000010011011..." + } + } + + + +### parse + +```python +def parse(data: str, raw: bool = False, quiet: bool = False) -> Dict +``` + +Main text parsing function + +Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + +Returns: + + Dictionary. Raw or processed structured data. + +### Parser Information +Compatibility: linux, darwin, cygwin, win32, aix, freebsd + +Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/jc/parsers/ip_address.py b/jc/parsers/ip_address.py index 2715b39a7..1af86f7aa 100644 --- a/jc/parsers/ip_address.py +++ b/jc/parsers/ip_address.py @@ -1,5 +1,9 @@ """jc - JSON Convert IP Address string parser +Accepts standard and integer IP address notation for both IPv4 and IPv6 +addresses. CIDR subnet mask and Scope ID is also allowed for standard +notation. See examples below. + Usage (cli): $ echo '192.168.1.1' | jc --ip-address @@ -11,21 +15,449 @@ Schema: - [ - { - "ip_address": string, - "bar": boolean, - "baz": integer + { + "version": integer, + "max_prefix_length": integer, + "ip": string, + "ip_compressed": string, + "ip_exploded": string, + "scope_id": string/null, + "ipv4_mapped": string/null, + "six_to_four": string/null, + "teredo_client": string/null, + "teredo_server": string/null, + "dns_ptr": string, + "network": string, + "broadcast": string, + "hostmask": string, + "netmask": string, + "cidr_netmask": integer, + "hosts": integer, + "first_host": string, + "last_host": string, + "is_multicast": boolean, + "is_private": boolean, + "is_global": boolean, + "is_link_local": boolean, + "is_loopback": boolean, + "is_reserved": boolean, + "is_unspecified": boolean, + "int": { + "ip": integer, + "network": integer, + "broadcast": integer, + "first_host": integer, + "last_host": integer + }, + "hex": { + "ip": string, + "network": string, + "broadcast": string, + "hostmask": string, + "netmask": string, + "first_host": string, + "last_host": string + }, + "bin": { + "ip": string, + "network": string, + "broadcast": string, + "hostmask": string, + "netmask": string, + "first_host": string, + "last_host": string } - ] + } Examples: - $ ip_address | jc --ip_address -p - [] - - $ ip_address | jc --ip_address -p -r - [] + $ echo 192.168.2.10/24 | jc --ip-address -p + { + "version": 4, + "max_prefix_length": 32, + "ip": "192.168.2.10", + "ip_compressed": "192.168.2.10/24", + "ip_exploded": "192.168.2.10/24", + "scope_id": null, + "ipv4_mapped": null, + "six_to_four": null, + "teredo_client": null, + "teredo_server": null, + "dns_ptr": "10.2.168.192.in-addr.arpa", + "network": "192.168.2.0", + "broadcast": "192.168.2.255", + "hostmask": "0.0.0.255", + "netmask": "255.255.255.0", + "cidr_netmask": 24, + "hosts": 254, + "first_host": "192.168.2.1", + "last_host": "192.168.2.254", + "is_multicast": false, + "is_private": true, + "is_global": false, + "is_link_local": false, + "is_loopback": false, + "is_reserved": false, + "is_unspecified": false, + "int": { + "ip": 3232236042, + "network": 3232236032, + "broadcast": 3232236287, + "first_host": 3232236033, + "last_host": 3232236286 + }, + "hex": { + "ip": "c0:a8:02:0a", + "network": "c0:a8:02:00", + "broadcast": "c0:a8:02:ff", + "hostmask": "00:00:00:ff", + "netmask": "ff:ff:ff:00", + "first_host": "c0:a8:02:01", + "last_host": "c0:a8:02:fe" + }, + "bin": { + "ip": "11000000101010000000001000001010", + "network": "11000000101010000000001000000000", + "broadcast": "11000000101010000000001011111111", + "hostmask": "00000000000000000000000011111111", + "netmask": "11111111111111111111111100000000", + "first_host": "11000000101010000000001000000001", + "last_host": "11000000101010000000001011111110" + } + } + + $ echo 3232236042 | jc --ip-address -p + { + "version": 4, + "max_prefix_length": 32, + "ip": "192.168.2.10", + "ip_compressed": "192.168.2.10/32", + "ip_exploded": "192.168.2.10/32", + "scope_id": null, + "ipv4_mapped": null, + "six_to_four": null, + "teredo_client": null, + "teredo_server": null, + "dns_ptr": "10.2.168.192.in-addr.arpa", + "network": "192.168.2.10", + "broadcast": "192.168.2.10", + "hostmask": "0.0.0.0", + "netmask": "255.255.255.255", + "cidr_netmask": 32, + "hosts": 1, + "first_host": "192.168.2.10", + "last_host": "192.168.2.10", + "is_multicast": false, + "is_private": true, + "is_global": false, + "is_link_local": false, + "is_loopback": false, + "is_reserved": false, + "is_unspecified": false, + "int": { + "ip": 3232236042, + "network": 3232236042, + "broadcast": 3232236042, + "first_host": 3232236042, + "last_host": 3232236042 + }, + "hex": { + "ip": "c0:a8:02:0a", + "network": "c0:a8:02:0a", + "broadcast": "c0:a8:02:0a", + "hostmask": "00:00:00:00", + "netmask": "ff:ff:ff:ff", + "first_host": "c0:a8:02:0a", + "last_host": "c0:a8:02:0a" + }, + "bin": { + "ip": "11000000101010000000001000001010", + "network": "11000000101010000000001000001010", + "broadcast": "11000000101010000000001000001010", + "hostmask": "00000000000000000000000000000000", + "netmask": "11111111111111111111111111111111", + "first_host": "11000000101010000000001000001010", + "last_host": "11000000101010000000001000001010" + } + } + + $ echo 127:0:de::1%128/96 | jc --ip-address -p + { + "version": 6, + "max_prefix_length": 128, + "ip": "127:0:de::1", + "ip_compressed": "127:0:de::1%128/96", + "ip_exploded": "0127:0000:00de:0000:0000:0000:0000:0001/96", + "scope_id": "128", + "ipv4_mapped": null, + "six_to_four": null, + "teredo_client": null, + "teredo_server": null, + "dns_ptr": "1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.....0.7.2.1.0.ip6.arpa", + "network": "127:0:de::", + "broadcast": "127:0:de::ffff:ffff", + "hostmask": "::ffff:ffff", + "netmask": "ffff:ffff:ffff:ffff:ffff:ffff::", + "cidr_netmask": 96, + "hosts": 4294967294, + "first_host": "127:0:de::1", + "last_host": "127:0:de::ffff:fffe", + "is_multicast": false, + "is_private": false, + "is_global": true, + "is_link_local": false, + "is_loopback": false, + "is_reserved": true, + "is_unspecified": false, + "int": { + "ip": 1531727573536155682370944093904699393, + "network": 1531727573536155682370944093904699392, + "broadcast": 1531727573536155682370944098199666687, + "first_host": 1531727573536155682370944093904699393, + "last_host": 1531727573536155682370944098199666686 + }, + "hex": { + "ip": "01:27:00:00:00:de:00:00:00:00:00:00:00:00:00:01", + "network": "01:27:00:00:00:de:00:00:00:00:00:00:00:00:00:00", + "broadcast": "01:27:00:00:00:de:00:00:00:00:00:00:ff:ff:ff:ff", + "hostmask": "00:00:00:00:00:00:00:00:00:00:00:00:ff:ff:ff:ff", + "netmask": "ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:00:00:00:00", + "first_host": "01:27:00:00:00:de:00:00:00:00:00:00:00:00:00:01", + "last_host": "01:27:00:00:00:de:00:00:00:00:00:00:ff:ff:ff:fe" + }, + "bin": { + "ip": "000000010010011100000000000000000000000011011110000000...", + "network": "0000000100100111000000000000000000000000110111100...", + "broadcast": "00000001001001110000000000000000000000001101111...", + "hostmask": "000000000000000000000000000000000000000000000000...", + "netmask": "1111111111111111111111111111111111111111111111111...", + "first_host": "0000000100100111000000000000000000000000110111...", + "last_host": "00000001001001110000000000000000000000001101111..." + } + } + + $ echo 1531727573536155682370944093904699393 | jc --ip-address -p + { + "version": 6, + "max_prefix_length": 128, + "ip": "127:0:de::1", + "ip_compressed": "127:0:de::1/128", + "ip_exploded": "0127:0000:00de:0000:0000:0000:0000:0001/128", + "scope_id": null, + "ipv4_mapped": null, + "six_to_four": null, + "teredo_client": null, + "teredo_server": null, + "dns_ptr": "1.0.0.0.0.0.0....0.0.0.e.d.0.0.0.0.0.0.7.2.1.0.ip6.arpa", + "network": "127:0:de::1", + "broadcast": "127:0:de::1", + "hostmask": "::", + "netmask": "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", + "cidr_netmask": 128, + "hosts": 1, + "first_host": "127:0:de::1", + "last_host": "127:0:de::1", + "is_multicast": false, + "is_private": false, + "is_global": true, + "is_link_local": false, + "is_loopback": false, + "is_reserved": true, + "is_unspecified": false, + "int": { + "ip": 1531727573536155682370944093904699393, + "network": 1531727573536155682370944093904699393, + "broadcast": 1531727573536155682370944093904699393, + "first_host": 1531727573536155682370944093904699393, + "last_host": 1531727573536155682370944093904699393 + }, + "hex": { + "ip": "01:27:00:00:00:de:00:00:00:00:00:00:00:00:00:01", + "network": "01:27:00:00:00:de:00:00:00:00:00:00:00:00:00:01", + "broadcast": "01:27:00:00:00:de:00:00:00:00:00:00:00:00:00:01", + "hostmask": "00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00", + "netmask": "ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff", + "first_host": "01:27:00:00:00:de:00:00:00:00:00:00:00:00:00:01", + "last_host": "01:27:00:00:00:de:00:00:00:00:00:00:00:00:00:01" + }, + "bin": { + "ip": "0000000100100111000000000000000000000000110111100000000...", + "network": "00000001001001110000000000000000000000001101111000...", + "broadcast": "000000010010011100000000000000000000000011011110...", + "hostmask": "0000000000000000000000000000000000000000000000000...", + "netmask": "11111111111111111111111111111111111111111111111111...", + "first_host": "00000001001001110000000000000000000000001101111...", + "last_host": "000000010010011100000000000000000000000011011110..." + } + } + + # IPv4 Mapped Address + $ echo ::FFFF:192.168.1.35 | jc --ip-address -p + { + "version": 6, + "max_prefix_length": 128, + "ip": "::ffff:c0a8:123", + "ip_compressed": "::ffff:c0a8:123/128", + "ip_exploded": "0000:0000:0000:0000:0000:ffff:c0a8:0123/128", + "scope_id": null, + "ipv4_mapped": "192.168.1.35", + "six_to_four": null, + "teredo_client": null, + "teredo_server": null, + "dns_ptr": "3.2.1.0.8.a.0.c.f.f.f.f.0.0.0....0.0.0.0.0.0.0.ip6.arpa", + "network": "::ffff:c0a8:123", + "broadcast": "::ffff:c0a8:123", + "hostmask": "::", + "netmask": "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", + "cidr_netmask": 128, + "hosts": 1, + "first_host": "::ffff:c0a8:123", + "last_host": "::ffff:c0a8:123", + "is_multicast": false, + "is_private": true, + "is_global": false, + "is_link_local": false, + "is_loopback": false, + "is_reserved": true, + "is_unspecified": false, + "int": { + "ip": 281473913979171, + "network": 281473913979171, + "broadcast": 281473913979171, + "first_host": 281473913979171, + "last_host": 281473913979171 + }, + "hex": { + "ip": "00:00:00:00:00:00:00:00:00:00:ff:ff:c0:a8:01:23", + "network": "00:00:00:00:00:00:00:00:00:00:ff:ff:c0:a8:01:23", + "broadcast": "00:00:00:00:00:00:00:00:00:00:ff:ff:c0:a8:01:23", + "hostmask": "00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00", + "netmask": "ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff", + "first_host": "00:00:00:00:00:00:00:00:00:00:ff:ff:c0:a8:01:23", + "last_host": "00:00:00:00:00:00:00:00:00:00:ff:ff:c0:a8:01:23" + }, + "bin": { + "ip": "0000000000000000000000000000000000000000000000000000000...", + "network": "00000000000000000000000000000000000000000000000000...", + "broadcast": "000000000000000000000000000000000000000000000000...", + "hostmask": "0000000000000000000000000000000000000000000000000...", + "netmask": "11111111111111111111111111111111111111111111111111...", + "first_host": "00000000000000000000000000000000000000000000000...", + "last_host": "000000000000000000000000000000000000000000000000..." + } + } + + # 6to4 Address + $ echo 2002:c000:204::/48 | jc --ip-address -p + { + "version": 6, + "max_prefix_length": 128, + "ip": "2002:c000:204::", + "ip_compressed": "2002:c000:204::/48", + "ip_exploded": "2002:c000:0204:0000:0000:0000:0000:0000/48", + "scope_id": null, + "ipv4_mapped": null, + "six_to_four": "192.0.2.4", + "teredo_client": null, + "teredo_server": null, + "dns_ptr": "0.0.0.0.0.0.0.0......0.4.0.2.0.0.0.0.c.2.0.0.2.ip6.arpa", + "network": "2002:c000:204::", + "broadcast": "2002:c000:204:ffff:ffff:ffff:ffff:ffff", + "hostmask": "::ffff:ffff:ffff:ffff:ffff", + "netmask": "ffff:ffff:ffff::", + "cidr_netmask": 48, + "hosts": 1208925819614629174706174, + "first_host": "2002:c000:204::1", + "last_host": "2002:c000:204:ffff:ffff:ffff:ffff:fffe", + "is_multicast": false, + "is_private": false, + "is_global": true, + "is_link_local": false, + "is_loopback": false, + "is_reserved": false, + "is_unspecified": false, + "int": { + "ip": 42549574682102084431821433448024768512, + "network": 42549574682102084431821433448024768512, + "broadcast": 42549574682103293357641048077199474687, + "first_host": 42549574682102084431821433448024768513, + "last_host": 42549574682103293357641048077199474686 + }, + "hex": { + "ip": "20:02:c0:00:02:04:00:00:00:00:00:00:00:00:00:00", + "network": "20:02:c0:00:02:04:00:00:00:00:00:00:00:00:00:00", + "broadcast": "20:02:c0:00:02:04:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff", + "hostmask": "00:00:00:00:00:00:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff", + "netmask": "ff:ff:ff:ff:ff:ff:00:00:00:00:00:00:00:00:00:00", + "first_host": "20:02:c0:00:02:04:00:00:00:00:00:00:00:00:00:01", + "last_host": "20:02:c0:00:02:04:ff:ff:ff:ff:ff:ff:ff:ff:ff:fe" + }, + "bin": { + "ip": "0010000000000010110000000000000000000010000001000000000...", + "network": "00100000000000101100000000000000000000100000010000...", + "broadcast": "001000000000001011000000000000000000001000000100...", + "hostmask": "0000000000000000000000000000000000000000000000001...", + "netmask": "11111111111111111111111111111111111111111111111100...", + "first_host": "00100000000000101100000000000000000000100000010...", + "last_host": "001000000000001011000000000000000000001000000100..." + } + } + + # Teredo Address + % echo 2001:0000:4136:e378:8000:63bf:3fff:fdd2 | jc --ip-address -p + { + "version": 6, + "max_prefix_length": 128, + "ip": "2001:0:4136:e378:8000:63bf:3fff:fdd2", + "ip_compressed": "2001:0:4136:e378:8000:63bf:3fff:fdd2/128", + "ip_exploded": "2001:0000:4136:e378:8000:63bf:3fff:fdd2/128", + "scope_id": null, + "ipv4_mapped": null, + "six_to_four": null, + "teredo_client": "192.0.2.45", + "teredo_server": "65.54.227.120", + "dns_ptr": "2.d.d.f.f.f.f.3.f.b.3.6.0.0.0....0.0.0.1.0.0.2.ip6.arpa", + "network": "2001:0:4136:e378:8000:63bf:3fff:fdd2", + "broadcast": "2001:0:4136:e378:8000:63bf:3fff:fdd2", + "hostmask": "::", + "netmask": "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", + "cidr_netmask": 128, + "hosts": 1, + "first_host": "2001:0:4136:e378:8000:63bf:3fff:fdd2", + "last_host": "2001:0:4136:e378:8000:63bf:3fff:fdd2", + "is_multicast": false, + "is_private": true, + "is_global": false, + "is_link_local": false, + "is_loopback": false, + "is_reserved": false, + "is_unspecified": false, + "int": { + "ip": 42540488182158724593221357832373272018, + "network": 42540488182158724593221357832373272018, + "broadcast": 42540488182158724593221357832373272018, + "first_host": 42540488182158724593221357832373272018, + "last_host": 42540488182158724593221357832373272018 + }, + "hex": { + "ip": "20:01:00:00:41:36:e3:78:80:00:63:bf:3f:ff:fd:d2", + "network": "20:01:00:00:41:36:e3:78:80:00:63:bf:3f:ff:fd:d2", + "broadcast": "20:01:00:00:41:36:e3:78:80:00:63:bf:3f:ff:fd:d2", + "hostmask": "00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00", + "netmask": "ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff", + "first_host": "20:01:00:00:41:36:e3:78:80:00:63:bf:3f:ff:fd:d2", + "last_host": "20:01:00:00:41:36:e3:78:80:00:63:bf:3f:ff:fd:d2" + }, + "bin": { + "ip": "001000000000000100000000000000000100000100110110111000...", + "network": "0010000000000001000000000000000001000001001101101...", + "broadcast": "00100000000000010000000000000000010000010011011...", + "hostmask": "000000000000000000000000000000000000000000000000...", + "netmask": "1111111111111111111111111111111111111111111111111...", + "first_host": "0010000000000001000000000000000001000001001101...", + "last_host": "00100000000000010000000000000000010000010011011..." + } + } """ from typing import Dict import binascii @@ -97,7 +529,13 @@ def parse( if jc.utils.has_data(data): - interface = ipaddress.ip_interface(data.strip()) + # Accept IPs entered as integer notation + try: + data = int(data) # type: ignore + except Exception: + data = data.strip() + + interface = ipaddress.ip_interface(data) network_string = str(interface.network).split('/')[0] network_cidr = int(str(interface.with_prefixlen).split('/')[1]) network = ipaddress.ip_network(f'{network_string}/{network_cidr}') @@ -170,6 +608,13 @@ def parse( 'is_loopback': interface.is_loopback, 'is_reserved': interface.is_reserved, 'is_unspecified': interface.is_unspecified, + 'int': { + 'ip': int(interface), + 'network': int(ipaddress.ip_address(network_string)), + 'broadcast': int(ipaddress.ip_address(broadcast_string)), + 'first_host': int(ipaddress.ip_address(first_host)), + 'last_host': int(ipaddress.ip_address(last_host)) + }, 'hex': { 'ip': _b2a(bare_ip.packed), 'network': _b2a(ipaddress.ip_address(network_string).packed), diff --git a/man/jc.1 b/man/jc.1 index 60b8b8a18..7bd6e7055 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -1,4 +1,4 @@ -.TH jc 1 2022-07-23 1.20.4 "JSON Convert" +.TH jc 1 2022-07-27 1.20.5 "JSON Convert" .SH NAME \fBjc\fP \- JSON Convert JSONifies the output of many CLI tools and file-types .SH SYNOPSIS @@ -222,6 +222,11 @@ INI file parser \fB--iostat-s\fP `iostat` command streaming parser +.TP +.B +\fB--ip-address\fP +IP Address string parser + .TP .B \fB--iptables\fP From 1df662102c9a3c7b6c0c8a06db7ea88880de92dc Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 28 Jul 2022 07:30:26 -0700 Subject: [PATCH 011/116] remove subnet on compressed and exploded --- jc/parsers/ip_address.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/jc/parsers/ip_address.py b/jc/parsers/ip_address.py index 1af86f7aa..253370ccd 100644 --- a/jc/parsers/ip_address.py +++ b/jc/parsers/ip_address.py @@ -76,8 +76,8 @@ "version": 4, "max_prefix_length": 32, "ip": "192.168.2.10", - "ip_compressed": "192.168.2.10/24", - "ip_exploded": "192.168.2.10/24", + "ip_compressed": "192.168.2.10", + "ip_exploded": "192.168.2.10", "scope_id": null, "ipv4_mapped": null, "six_to_four": null, @@ -131,8 +131,8 @@ "version": 4, "max_prefix_length": 32, "ip": "192.168.2.10", - "ip_compressed": "192.168.2.10/32", - "ip_exploded": "192.168.2.10/32", + "ip_compressed": "192.168.2.10", + "ip_exploded": "192.168.2.10", "scope_id": null, "ipv4_mapped": null, "six_to_four": null, @@ -186,8 +186,8 @@ "version": 6, "max_prefix_length": 128, "ip": "127:0:de::1", - "ip_compressed": "127:0:de::1%128/96", - "ip_exploded": "0127:0000:00de:0000:0000:0000:0000:0001/96", + "ip_compressed": "127:0:de::1%128", + "ip_exploded": "0127:0000:00de:0000:0000:0000:0000:0001", "scope_id": "128", "ipv4_mapped": null, "six_to_four": null, @@ -241,8 +241,8 @@ "version": 6, "max_prefix_length": 128, "ip": "127:0:de::1", - "ip_compressed": "127:0:de::1/128", - "ip_exploded": "0127:0000:00de:0000:0000:0000:0000:0001/128", + "ip_compressed": "127:0:de::1", + "ip_exploded": "0127:0000:00de:0000:0000:0000:0000:0001", "scope_id": null, "ipv4_mapped": null, "six_to_four": null, @@ -297,8 +297,8 @@ "version": 6, "max_prefix_length": 128, "ip": "::ffff:c0a8:123", - "ip_compressed": "::ffff:c0a8:123/128", - "ip_exploded": "0000:0000:0000:0000:0000:ffff:c0a8:0123/128", + "ip_compressed": "::ffff:c0a8:123", + "ip_exploded": "0000:0000:0000:0000:0000:ffff:c0a8:0123", "scope_id": null, "ipv4_mapped": "192.168.1.35", "six_to_four": null, @@ -353,8 +353,8 @@ "version": 6, "max_prefix_length": 128, "ip": "2002:c000:204::", - "ip_compressed": "2002:c000:204::/48", - "ip_exploded": "2002:c000:0204:0000:0000:0000:0000:0000/48", + "ip_compressed": "2002:c000:204::", + "ip_exploded": "2002:c000:0204:0000:0000:0000:0000:0000", "scope_id": null, "ipv4_mapped": null, "six_to_four": "192.0.2.4", @@ -409,8 +409,8 @@ "version": 6, "max_prefix_length": 128, "ip": "2001:0:4136:e378:8000:63bf:3fff:fdd2", - "ip_compressed": "2001:0:4136:e378:8000:63bf:3fff:fdd2/128", - "ip_exploded": "2001:0000:4136:e378:8000:63bf:3fff:fdd2/128", + "ip_compressed": "2001:0:4136:e378:8000:63bf:3fff:fdd2", + "ip_exploded": "2001:0000:4136:e378:8000:63bf:3fff:fdd2", "scope_id": null, "ipv4_mapped": null, "six_to_four": null, @@ -585,8 +585,8 @@ def parse( 'version': int(interface.version), 'max_prefix_length': interface.max_prefixlen, 'ip': bare_ip_string, - 'ip_compressed': str(interface.compressed), - 'ip_exploded': str(interface.exploded), + 'ip_compressed': bare_ip.compressed, + 'ip_exploded': bare_ip.exploded, 'scope_id': scope_id, 'ipv4_mapped': ipv4_mapped, 'six_to_four': sixtofour, From 92956cb1a753a7139eb3e5f870a185f45ab31a41 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 28 Jul 2022 07:31:30 -0700 Subject: [PATCH 012/116] doc update --- docs/parsers/ip_address.md | 28 ++++++++++++++-------------- man/jc.1 | 2 +- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/docs/parsers/ip_address.md b/docs/parsers/ip_address.md index e98bda7a6..a67589fe1 100644 --- a/docs/parsers/ip_address.md +++ b/docs/parsers/ip_address.md @@ -81,8 +81,8 @@ Examples: "version": 4, "max_prefix_length": 32, "ip": "192.168.2.10", - "ip_compressed": "192.168.2.10/24", - "ip_exploded": "192.168.2.10/24", + "ip_compressed": "192.168.2.10", + "ip_exploded": "192.168.2.10", "scope_id": null, "ipv4_mapped": null, "six_to_four": null, @@ -136,8 +136,8 @@ Examples: "version": 4, "max_prefix_length": 32, "ip": "192.168.2.10", - "ip_compressed": "192.168.2.10/32", - "ip_exploded": "192.168.2.10/32", + "ip_compressed": "192.168.2.10", + "ip_exploded": "192.168.2.10", "scope_id": null, "ipv4_mapped": null, "six_to_four": null, @@ -191,8 +191,8 @@ Examples: "version": 6, "max_prefix_length": 128, "ip": "127:0:de::1", - "ip_compressed": "127:0:de::1%128/96", - "ip_exploded": "0127:0000:00de:0000:0000:0000:0000:0001/96", + "ip_compressed": "127:0:de::1%128", + "ip_exploded": "0127:0000:00de:0000:0000:0000:0000:0001", "scope_id": "128", "ipv4_mapped": null, "six_to_four": null, @@ -246,8 +246,8 @@ Examples: "version": 6, "max_prefix_length": 128, "ip": "127:0:de::1", - "ip_compressed": "127:0:de::1/128", - "ip_exploded": "0127:0000:00de:0000:0000:0000:0000:0001/128", + "ip_compressed": "127:0:de::1", + "ip_exploded": "0127:0000:00de:0000:0000:0000:0000:0001", "scope_id": null, "ipv4_mapped": null, "six_to_four": null, @@ -302,8 +302,8 @@ Examples: "version": 6, "max_prefix_length": 128, "ip": "::ffff:c0a8:123", - "ip_compressed": "::ffff:c0a8:123/128", - "ip_exploded": "0000:0000:0000:0000:0000:ffff:c0a8:0123/128", + "ip_compressed": "::ffff:c0a8:123", + "ip_exploded": "0000:0000:0000:0000:0000:ffff:c0a8:0123", "scope_id": null, "ipv4_mapped": "192.168.1.35", "six_to_four": null, @@ -358,8 +358,8 @@ Examples: "version": 6, "max_prefix_length": 128, "ip": "2002:c000:204::", - "ip_compressed": "2002:c000:204::/48", - "ip_exploded": "2002:c000:0204:0000:0000:0000:0000:0000/48", + "ip_compressed": "2002:c000:204::", + "ip_exploded": "2002:c000:0204:0000:0000:0000:0000:0000", "scope_id": null, "ipv4_mapped": null, "six_to_four": "192.0.2.4", @@ -414,8 +414,8 @@ Examples: "version": 6, "max_prefix_length": 128, "ip": "2001:0:4136:e378:8000:63bf:3fff:fdd2", - "ip_compressed": "2001:0:4136:e378:8000:63bf:3fff:fdd2/128", - "ip_exploded": "2001:0000:4136:e378:8000:63bf:3fff:fdd2/128", + "ip_compressed": "2001:0:4136:e378:8000:63bf:3fff:fdd2", + "ip_exploded": "2001:0000:4136:e378:8000:63bf:3fff:fdd2", "scope_id": null, "ipv4_mapped": null, "six_to_four": null, diff --git a/man/jc.1 b/man/jc.1 index 7bd6e7055..3bdec263a 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -1,4 +1,4 @@ -.TH jc 1 2022-07-27 1.20.5 "JSON Convert" +.TH jc 1 2022-07-28 1.20.5 "JSON Convert" .SH NAME \fBjc\fP \- JSON Convert JSONifies the output of many CLI tools and file-types .SH SYNOPSIS From 6b862dc860055b055af13517fb5eb4297e3a5b89 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 28 Jul 2022 09:00:56 -0700 Subject: [PATCH 013/116] object cleanup --- jc/parsers/ip_address.py | 66 +++++++++++++++++++++++++--------------- 1 file changed, 41 insertions(+), 25 deletions(-) diff --git a/jc/parsers/ip_address.py b/jc/parsers/ip_address.py index 253370ccd..901aa3b10 100644 --- a/jc/parsers/ip_address.py +++ b/jc/parsers/ip_address.py @@ -459,7 +459,7 @@ } } """ -from typing import Dict +from typing import Dict, Union import binascii import ipaddress import jc.utils @@ -468,7 +468,7 @@ class info(): """Provides parser metadata (version, author, etc.)""" version = '1.0' - description = 'IP Address string parser' + description = 'IPv4 and IPv6 Address string parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd'] @@ -504,6 +504,10 @@ def _b2a(byte_string: bytes) -> str: return colon_seperated +def _bin_format(ip: Union[ipaddress.IPv4Address,ipaddress.IPv6Address], length: int) -> str: + return format(int(ip), '0>' + str(length) +'b') + + def parse( data: str, raw: bool = False, @@ -536,12 +540,21 @@ def parse( data = data.strip() interface = ipaddress.ip_interface(data) + network_string = str(interface.network).split('/')[0] network_cidr = int(str(interface.with_prefixlen).split('/')[1]) network = ipaddress.ip_network(f'{network_string}/{network_cidr}') + network_ipobj = ipaddress.ip_address(network_string) + broadcast_string = str(network.broadcast_address) + broadcast_ipobj = ipaddress.ip_address(broadcast_string) + hostmask_string = str(interface.with_hostmask).split('/')[1] + hostmask_ipobj = ipaddress.ip_address(hostmask_string) + netmask_string = str(interface.with_netmask).split('/')[1] + netmask_ipobj = ipaddress.ip_address(netmask_string) + bare_ip_string = str(interface.ip) bare_ip = ipaddress.ip_address(bare_ip_string) ip_ptr = bare_ip.reverse_pointer @@ -564,8 +577,8 @@ def parse( interface.version == 4 and network_cidr == 31, interface.version == 6 and network_cidr == 127 )): - first_host = str(ipaddress.ip_address(network_string)) - last_host = str(ipaddress.ip_address(broadcast_string)) + first_host = network_string + last_host = broadcast_string hosts = 2 elif any(( @@ -577,12 +590,15 @@ def parse( hosts = 1 else: - first_host = str(ipaddress.ip_address(network_string) + 1) - last_host = str(ipaddress.ip_address(broadcast_string) - 1) - hosts = int(ipaddress.ip_address(broadcast_string) - 1) - int(ipaddress.ip_address(network_string) + 1) + 1 + first_host = str(network_ipobj + 1) + last_host = str(broadcast_ipobj - 1) + hosts = int(broadcast_ipobj - 1) - int(network_ipobj + 1) + 1 + + first_host_ipobj = ipaddress.ip_address(first_host) + last_host_ipobj = ipaddress.ip_address(last_host) raw_output = { - 'version': int(interface.version), + 'version': interface.version, 'max_prefix_length': interface.max_prefixlen, 'ip': bare_ip_string, 'ip_compressed': bare_ip.compressed, @@ -610,28 +626,28 @@ def parse( 'is_unspecified': interface.is_unspecified, 'int': { 'ip': int(interface), - 'network': int(ipaddress.ip_address(network_string)), - 'broadcast': int(ipaddress.ip_address(broadcast_string)), - 'first_host': int(ipaddress.ip_address(first_host)), - 'last_host': int(ipaddress.ip_address(last_host)) + 'network': int(network_ipobj), + 'broadcast': int(broadcast_ipobj), + 'first_host': int(first_host_ipobj), + 'last_host': int(last_host_ipobj) }, 'hex': { 'ip': _b2a(bare_ip.packed), - 'network': _b2a(ipaddress.ip_address(network_string).packed), - 'broadcast': _b2a(ipaddress.ip_address(broadcast_string).packed), - 'hostmask': _b2a(ipaddress.ip_address(hostmask_string).packed), - 'netmask': _b2a(ipaddress.ip_address(netmask_string).packed), - 'first_host': _b2a(ipaddress.ip_address(first_host).packed), - 'last_host': _b2a(ipaddress.ip_address(last_host).packed) + 'network': _b2a(network_ipobj.packed), + 'broadcast': _b2a(broadcast_ipobj.packed), + 'hostmask': _b2a(hostmask_ipobj.packed), + 'netmask': _b2a(netmask_ipobj.packed), + 'first_host': _b2a(first_host_ipobj.packed), + 'last_host': _b2a(last_host_ipobj.packed) }, 'bin': { - 'ip': format(int(bare_ip), '0>' + str(interface.max_prefixlen) +'b'), - 'network': format(int(ipaddress.ip_address(network_string)), '0>' + str(interface.max_prefixlen) +'b'), - 'broadcast': format(int(ipaddress.ip_address(broadcast_string)), '0>' + str(interface.max_prefixlen) +'b'), - 'hostmask': format(int(ipaddress.ip_address(hostmask_string)), '0>' + str(interface.max_prefixlen) +'b'), - 'netmask': format(int(ipaddress.ip_address(netmask_string)), '0>' + str(interface.max_prefixlen) +'b'), - 'first_host': format(int(ipaddress.ip_address(first_host)), '0>' + str(interface.max_prefixlen) +'b'), - 'last_host': format(int(ipaddress.ip_address(last_host)), '0>' + str(interface.max_prefixlen) +'b'), + 'ip': _bin_format(bare_ip, interface.max_prefixlen), + 'network': _bin_format(network_ipobj, interface.max_prefixlen), + 'broadcast': _bin_format(broadcast_ipobj, interface.max_prefixlen), + 'hostmask': _bin_format(hostmask_ipobj, interface.max_prefixlen), + 'netmask': _bin_format(netmask_ipobj, interface.max_prefixlen), + 'first_host': _bin_format(first_host_ipobj, interface.max_prefixlen), + 'last_host': _bin_format(last_host_ipobj, interface.max_prefixlen) } } From c8c3d5dc3d86adc2af5b4161745d909f5e3d421b Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 28 Jul 2022 09:03:49 -0700 Subject: [PATCH 014/116] formatting --- README.md | 2 +- completions/jc_zsh_completion.sh | 2 +- docs/parsers/ip_address.md | 2 +- jc/parsers/ip_address.py | 2 +- man/jc.1 | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 092f1100e..c61b0c026 100644 --- a/README.md +++ b/README.md @@ -189,7 +189,7 @@ option. | ` --ini` | INI file parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/ini) | | ` --iostat` | `iostat` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/iostat) | | ` --iostat-s` | `iostat` command streaming parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/iostat_s) | -| ` --ip-address` | IP Address string parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/ip_address) | +| ` --ip-address` | IPv4 and IPv6 Address string parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/ip_address) | | ` --iptables` | `iptables` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/iptables) | | ` --iso-datetime` | ISO 8601 Datetime string parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/iso_datetime) | | ` --iw-scan` | `iw dev [device] scan` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/iw_scan) | diff --git a/completions/jc_zsh_completion.sh b/completions/jc_zsh_completion.sh index c65c4b905..f0aaae726 100644 --- a/completions/jc_zsh_completion.sh +++ b/completions/jc_zsh_completion.sh @@ -137,7 +137,7 @@ _jc() { '--ini:INI file parser' '--iostat:`iostat` command parser' '--iostat-s:`iostat` command streaming parser' - '--ip-address:IP Address string parser' + '--ip-address:IPv4 and IPv6 Address string parser' '--iptables:`iptables` command parser' '--iso-datetime:ISO 8601 Datetime string parser' '--iw-scan:`iw dev [device] scan` command parser' diff --git a/docs/parsers/ip_address.md b/docs/parsers/ip_address.md index a67589fe1..aa664e73f 100644 --- a/docs/parsers/ip_address.md +++ b/docs/parsers/ip_address.md @@ -409,7 +409,7 @@ Examples: } # Teredo Address - % echo 2001:0000:4136:e378:8000:63bf:3fff:fdd2 | jc --ip-address -p + $ echo 2001:0000:4136:e378:8000:63bf:3fff:fdd2 | jc --ip-address -p { "version": 6, "max_prefix_length": 128, diff --git a/jc/parsers/ip_address.py b/jc/parsers/ip_address.py index 901aa3b10..61fd61843 100644 --- a/jc/parsers/ip_address.py +++ b/jc/parsers/ip_address.py @@ -404,7 +404,7 @@ } # Teredo Address - % echo 2001:0000:4136:e378:8000:63bf:3fff:fdd2 | jc --ip-address -p + $ echo 2001:0000:4136:e378:8000:63bf:3fff:fdd2 | jc --ip-address -p { "version": 6, "max_prefix_length": 128, diff --git a/man/jc.1 b/man/jc.1 index 3bdec263a..5b2a46107 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -225,7 +225,7 @@ INI file parser .TP .B \fB--ip-address\fP -IP Address string parser +IPv4 and IPv6 Address string parser .TP .B From 3f74571dfc2febd373b0457ff9374eaaeffd4159 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 28 Jul 2022 09:11:02 -0700 Subject: [PATCH 015/116] formatting --- jc/parsers/ip_address.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jc/parsers/ip_address.py b/jc/parsers/ip_address.py index 61fd61843..03b7c18ca 100644 --- a/jc/parsers/ip_address.py +++ b/jc/parsers/ip_address.py @@ -504,7 +504,7 @@ def _b2a(byte_string: bytes) -> str: return colon_seperated -def _bin_format(ip: Union[ipaddress.IPv4Address,ipaddress.IPv6Address], length: int) -> str: +def _bin_format(ip: Union[ipaddress.IPv4Address, ipaddress.IPv6Address], length: int) -> str: return format(int(ip), '0>' + str(length) +'b') From 6625ade578ea50cdda9f05a3326b4067a79b4d28 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 28 Jul 2022 09:58:02 -0700 Subject: [PATCH 016/116] add ip_address tests --- tests/test_ip_address.py | 106 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 tests/test_ip_address.py diff --git a/tests/test_ip_address.py b/tests/test_ip_address.py new file mode 100644 index 000000000..056cd9a7a --- /dev/null +++ b/tests/test_ip_address.py @@ -0,0 +1,106 @@ +import unittest +import json +import jc.parsers.ip_address + + +class MyTests(unittest.TestCase): + + def test_ip_address_nodata(self): + """ + Test 'ip_address' with no data + """ + self.assertEqual(jc.parsers.ip_address.parse('', quiet=True), {}) + + + def test_ip_address_ipv4(self): + """ + Test ipv4 address string + """ + data = r'192.168.1.35' + expected = json.loads(r'''{"version":4,"max_prefix_length":32,"ip":"192.168.1.35","ip_compressed":"192.168.1.35","ip_exploded":"192.168.1.35","scope_id":null,"ipv4_mapped":null,"six_to_four":null,"teredo_client":null,"teredo_server":null,"dns_ptr":"35.1.168.192.in-addr.arpa","network":"192.168.1.35","broadcast":"192.168.1.35","hostmask":"0.0.0.0","netmask":"255.255.255.255","cidr_netmask":32,"hosts":1,"first_host":"192.168.1.35","last_host":"192.168.1.35","is_multicast":false,"is_private":true,"is_global":false,"is_link_local":false,"is_loopback":false,"is_reserved":false,"is_unspecified":false,"int":{"ip":3232235811,"network":3232235811,"broadcast":3232235811,"first_host":3232235811,"last_host":3232235811},"hex":{"ip":"c0:a8:01:23","network":"c0:a8:01:23","broadcast":"c0:a8:01:23","hostmask":"00:00:00:00","netmask":"ff:ff:ff:ff","first_host":"c0:a8:01:23","last_host":"c0:a8:01:23"},"bin":{"ip":"11000000101010000000000100100011","network":"11000000101010000000000100100011","broadcast":"11000000101010000000000100100011","hostmask":"00000000000000000000000000000000","netmask":"11111111111111111111111111111111","first_host":"11000000101010000000000100100011","last_host":"11000000101010000000000100100011"}}''') + self.assertEqual(jc.parsers.ip_address.parse(data, quiet=True), expected) + + + def test_ip_address_ipv4_cidr(self): + """ + Test CIDR ipv4 address string + """ + data = r'192.168.2.10/24' + expected = json.loads(r'''{"version":4,"max_prefix_length":32,"ip":"192.168.2.10","ip_compressed":"192.168.2.10","ip_exploded":"192.168.2.10","scope_id":null,"ipv4_mapped":null,"six_to_four":null,"teredo_client":null,"teredo_server":null,"dns_ptr":"10.2.168.192.in-addr.arpa","network":"192.168.2.0","broadcast":"192.168.2.255","hostmask":"0.0.0.255","netmask":"255.255.255.0","cidr_netmask":24,"hosts":254,"first_host":"192.168.2.1","last_host":"192.168.2.254","is_multicast":false,"is_private":true,"is_global":false,"is_link_local":false,"is_loopback":false,"is_reserved":false,"is_unspecified":false,"int":{"ip":3232236042,"network":3232236032,"broadcast":3232236287,"first_host":3232236033,"last_host":3232236286},"hex":{"ip":"c0:a8:02:0a","network":"c0:a8:02:00","broadcast":"c0:a8:02:ff","hostmask":"00:00:00:ff","netmask":"ff:ff:ff:00","first_host":"c0:a8:02:01","last_host":"c0:a8:02:fe"},"bin":{"ip":"11000000101010000000001000001010","network":"11000000101010000000001000000000","broadcast":"11000000101010000000001011111111","hostmask":"00000000000000000000000011111111","netmask":"11111111111111111111111100000000","first_host":"11000000101010000000001000000001","last_host":"11000000101010000000001011111110"}}''') + self.assertEqual(jc.parsers.ip_address.parse(data, quiet=True), expected) + + + def test_ip_address_ipv4_integer(self): + """ + Test ipv4 address integer string + """ + data = r'3232236042' + expected = json.loads(r'''{"version":4,"max_prefix_length":32,"ip":"192.168.2.10","ip_compressed":"192.168.2.10","ip_exploded":"192.168.2.10","scope_id":null,"ipv4_mapped":null,"six_to_four":null,"teredo_client":null,"teredo_server":null,"dns_ptr":"10.2.168.192.in-addr.arpa","network":"192.168.2.10","broadcast":"192.168.2.10","hostmask":"0.0.0.0","netmask":"255.255.255.255","cidr_netmask":32,"hosts":1,"first_host":"192.168.2.10","last_host":"192.168.2.10","is_multicast":false,"is_private":true,"is_global":false,"is_link_local":false,"is_loopback":false,"is_reserved":false,"is_unspecified":false,"int":{"ip":3232236042,"network":3232236042,"broadcast":3232236042,"first_host":3232236042,"last_host":3232236042},"hex":{"ip":"c0:a8:02:0a","network":"c0:a8:02:0a","broadcast":"c0:a8:02:0a","hostmask":"00:00:00:00","netmask":"ff:ff:ff:ff","first_host":"c0:a8:02:0a","last_host":"c0:a8:02:0a"},"bin":{"ip":"11000000101010000000001000001010","network":"11000000101010000000001000001010","broadcast":"11000000101010000000001000001010","hostmask":"00000000000000000000000000000000","netmask":"11111111111111111111111111111111","first_host":"11000000101010000000001000001010","last_host":"11000000101010000000001000001010"}}''') + self.assertEqual(jc.parsers.ip_address.parse(data, quiet=True), expected) + + + def test_ip_address_ipv6(self): + """ + Test ipv6 address string + """ + data = r'127:0:de::1' + expected = json.loads(r'''{"version":6,"max_prefix_length":128,"ip":"127:0:de::1","ip_compressed":"127:0:de::1","ip_exploded":"0127:0000:00de:0000:0000:0000:0000:0001","scope_id":null,"ipv4_mapped":null,"six_to_four":null,"teredo_client":null,"teredo_server":null,"dns_ptr":"1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.e.d.0.0.0.0.0.0.7.2.1.0.ip6.arpa","network":"127:0:de::1","broadcast":"127:0:de::1","hostmask":"::","netmask":"ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff","cidr_netmask":128,"hosts":1,"first_host":"127:0:de::1","last_host":"127:0:de::1","is_multicast":false,"is_private":false,"is_global":true,"is_link_local":false,"is_loopback":false,"is_reserved":true,"is_unspecified":false,"int":{"ip":1531727573536155682370944093904699393,"network":1531727573536155682370944093904699393,"broadcast":1531727573536155682370944093904699393,"first_host":1531727573536155682370944093904699393,"last_host":1531727573536155682370944093904699393},"hex":{"ip":"01:27:00:00:00:de:00:00:00:00:00:00:00:00:00:01","network":"01:27:00:00:00:de:00:00:00:00:00:00:00:00:00:01","broadcast":"01:27:00:00:00:de:00:00:00:00:00:00:00:00:00:01","hostmask":"00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00","netmask":"ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff","first_host":"01:27:00:00:00:de:00:00:00:00:00:00:00:00:00:01","last_host":"01:27:00:00:00:de:00:00:00:00:00:00:00:00:00:01"},"bin":{"ip":"00000001001001110000000000000000000000001101111000000000000000000000000000000000000000000000000000000000000000000000000000000001","network":"00000001001001110000000000000000000000001101111000000000000000000000000000000000000000000000000000000000000000000000000000000001","broadcast":"00000001001001110000000000000000000000001101111000000000000000000000000000000000000000000000000000000000000000000000000000000001","hostmask":"00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000","netmask":"11111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111","first_host":"00000001001001110000000000000000000000001101111000000000000000000000000000000000000000000000000000000000000000000000000000000001","last_host":"00000001001001110000000000000000000000001101111000000000000000000000000000000000000000000000000000000000000000000000000000000001"}}''') + self.assertEqual(jc.parsers.ip_address.parse(data, quiet=True), expected) + + + def test_ip_address_ipv6_cidr(self): + """ + Test CIDR ipv6 address string + """ + data = r'127:0:de::1/96' + expected = json.loads(r'''{"version":6,"max_prefix_length":128,"ip":"127:0:de::1","ip_compressed":"127:0:de::1","ip_exploded":"0127:0000:00de:0000:0000:0000:0000:0001","scope_id":null,"ipv4_mapped":null,"six_to_four":null,"teredo_client":null,"teredo_server":null,"dns_ptr":"1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.e.d.0.0.0.0.0.0.7.2.1.0.ip6.arpa","network":"127:0:de::","broadcast":"127:0:de::ffff:ffff","hostmask":"::ffff:ffff","netmask":"ffff:ffff:ffff:ffff:ffff:ffff::","cidr_netmask":96,"hosts":4294967294,"first_host":"127:0:de::1","last_host":"127:0:de::ffff:fffe","is_multicast":false,"is_private":false,"is_global":true,"is_link_local":false,"is_loopback":false,"is_reserved":true,"is_unspecified":false,"int":{"ip":1531727573536155682370944093904699393,"network":1531727573536155682370944093904699392,"broadcast":1531727573536155682370944098199666687,"first_host":1531727573536155682370944093904699393,"last_host":1531727573536155682370944098199666686},"hex":{"ip":"01:27:00:00:00:de:00:00:00:00:00:00:00:00:00:01","network":"01:27:00:00:00:de:00:00:00:00:00:00:00:00:00:00","broadcast":"01:27:00:00:00:de:00:00:00:00:00:00:ff:ff:ff:ff","hostmask":"00:00:00:00:00:00:00:00:00:00:00:00:ff:ff:ff:ff","netmask":"ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:00:00:00:00","first_host":"01:27:00:00:00:de:00:00:00:00:00:00:00:00:00:01","last_host":"01:27:00:00:00:de:00:00:00:00:00:00:ff:ff:ff:fe"},"bin":{"ip":"00000001001001110000000000000000000000001101111000000000000000000000000000000000000000000000000000000000000000000000000000000001","network":"00000001001001110000000000000000000000001101111000000000000000000000000000000000000000000000000000000000000000000000000000000000","broadcast":"00000001001001110000000000000000000000001101111000000000000000000000000000000000000000000000000011111111111111111111111111111111","hostmask":"00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000011111111111111111111111111111111","netmask":"11111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100000000000000000000000000000000","first_host":"00000001001001110000000000000000000000001101111000000000000000000000000000000000000000000000000000000000000000000000000000000001","last_host":"00000001001001110000000000000000000000001101111000000000000000000000000000000000000000000000000011111111111111111111111111111110"}}''') + self.assertEqual(jc.parsers.ip_address.parse(data, quiet=True), expected) + + + def test_ip_address_ipv6_cidr_scope(self): + """ + Test CIDR ipv6 address with scope string + """ + data = r'127:0:de::1%128/96' + expected = json.loads(r'''{"version":6,"max_prefix_length":128,"ip":"127:0:de::1","ip_compressed":"127:0:de::1","ip_exploded":"0127:0000:00de:0000:0000:0000:0000:0001","scope_id":"128","ipv4_mapped":null,"six_to_four":null,"teredo_client":null,"teredo_server":null,"dns_ptr":"1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.e.d.0.0.0.0.0.0.7.2.1.0.ip6.arpa","network":"127:0:de::","broadcast":"127:0:de::ffff:ffff","hostmask":"::ffff:ffff","netmask":"ffff:ffff:ffff:ffff:ffff:ffff::","cidr_netmask":96,"hosts":4294967294,"first_host":"127:0:de::1","last_host":"127:0:de::ffff:fffe","is_multicast":false,"is_private":false,"is_global":true,"is_link_local":false,"is_loopback":false,"is_reserved":true,"is_unspecified":false,"int":{"ip":1531727573536155682370944093904699393,"network":1531727573536155682370944093904699392,"broadcast":1531727573536155682370944098199666687,"first_host":1531727573536155682370944093904699393,"last_host":1531727573536155682370944098199666686},"hex":{"ip":"01:27:00:00:00:de:00:00:00:00:00:00:00:00:00:01","network":"01:27:00:00:00:de:00:00:00:00:00:00:00:00:00:00","broadcast":"01:27:00:00:00:de:00:00:00:00:00:00:ff:ff:ff:ff","hostmask":"00:00:00:00:00:00:00:00:00:00:00:00:ff:ff:ff:ff","netmask":"ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:00:00:00:00","first_host":"01:27:00:00:00:de:00:00:00:00:00:00:00:00:00:01","last_host":"01:27:00:00:00:de:00:00:00:00:00:00:ff:ff:ff:fe"},"bin":{"ip":"00000001001001110000000000000000000000001101111000000000000000000000000000000000000000000000000000000000000000000000000000000001","network":"00000001001001110000000000000000000000001101111000000000000000000000000000000000000000000000000000000000000000000000000000000000","broadcast":"00000001001001110000000000000000000000001101111000000000000000000000000000000000000000000000000011111111111111111111111111111111","hostmask":"00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000011111111111111111111111111111111","netmask":"11111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100000000000000000000000000000000","first_host":"00000001001001110000000000000000000000001101111000000000000000000000000000000000000000000000000000000000000000000000000000000001","last_host":"00000001001001110000000000000000000000001101111000000000000000000000000000000000000000000000000011111111111111111111111111111110"}}''') + self.assertEqual(jc.parsers.ip_address.parse(data, quiet=True), expected) + + + def test_ip_address_ipv6_integer(self): + """ + Test ipv6 address integer string + """ + data = r'1531727573536155682370944093904699393' + expected = json.loads(r'''{"version":6,"max_prefix_length":128,"ip":"127:0:de::1","ip_compressed":"127:0:de::1","ip_exploded":"0127:0000:00de:0000:0000:0000:0000:0001","scope_id":null,"ipv4_mapped":null,"six_to_four":null,"teredo_client":null,"teredo_server":null,"dns_ptr":"1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.e.d.0.0.0.0.0.0.7.2.1.0.ip6.arpa","network":"127:0:de::1","broadcast":"127:0:de::1","hostmask":"::","netmask":"ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff","cidr_netmask":128,"hosts":1,"first_host":"127:0:de::1","last_host":"127:0:de::1","is_multicast":false,"is_private":false,"is_global":true,"is_link_local":false,"is_loopback":false,"is_reserved":true,"is_unspecified":false,"int":{"ip":1531727573536155682370944093904699393,"network":1531727573536155682370944093904699393,"broadcast":1531727573536155682370944093904699393,"first_host":1531727573536155682370944093904699393,"last_host":1531727573536155682370944093904699393},"hex":{"ip":"01:27:00:00:00:de:00:00:00:00:00:00:00:00:00:01","network":"01:27:00:00:00:de:00:00:00:00:00:00:00:00:00:01","broadcast":"01:27:00:00:00:de:00:00:00:00:00:00:00:00:00:01","hostmask":"00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00","netmask":"ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff","first_host":"01:27:00:00:00:de:00:00:00:00:00:00:00:00:00:01","last_host":"01:27:00:00:00:de:00:00:00:00:00:00:00:00:00:01"},"bin":{"ip":"00000001001001110000000000000000000000001101111000000000000000000000000000000000000000000000000000000000000000000000000000000001","network":"00000001001001110000000000000000000000001101111000000000000000000000000000000000000000000000000000000000000000000000000000000001","broadcast":"00000001001001110000000000000000000000001101111000000000000000000000000000000000000000000000000000000000000000000000000000000001","hostmask":"00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000","netmask":"11111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111","first_host":"00000001001001110000000000000000000000001101111000000000000000000000000000000000000000000000000000000000000000000000000000000001","last_host":"00000001001001110000000000000000000000001101111000000000000000000000000000000000000000000000000000000000000000000000000000000001"}}''') + self.assertEqual(jc.parsers.ip_address.parse(data, quiet=True), expected) + + + def test_ip_address_ipv6_ipv4_mapped(self): + """ + Test ipv6 address with ipv4 mapped string + """ + data = r'::FFFF:192.168.1.35' + expected = json.loads(r'''{"version":6,"max_prefix_length":128,"ip":"::ffff:c0a8:123","ip_compressed":"::ffff:c0a8:123","ip_exploded":"0000:0000:0000:0000:0000:ffff:c0a8:0123","scope_id":null,"ipv4_mapped":"192.168.1.35","six_to_four":null,"teredo_client":null,"teredo_server":null,"dns_ptr":"3.2.1.0.8.a.0.c.f.f.f.f.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.ip6.arpa","network":"::ffff:c0a8:123","broadcast":"::ffff:c0a8:123","hostmask":"::","netmask":"ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff","cidr_netmask":128,"hosts":1,"first_host":"::ffff:c0a8:123","last_host":"::ffff:c0a8:123","is_multicast":false,"is_private":true,"is_global":false,"is_link_local":false,"is_loopback":false,"is_reserved":true,"is_unspecified":false,"int":{"ip":281473913979171,"network":281473913979171,"broadcast":281473913979171,"first_host":281473913979171,"last_host":281473913979171},"hex":{"ip":"00:00:00:00:00:00:00:00:00:00:ff:ff:c0:a8:01:23","network":"00:00:00:00:00:00:00:00:00:00:ff:ff:c0:a8:01:23","broadcast":"00:00:00:00:00:00:00:00:00:00:ff:ff:c0:a8:01:23","hostmask":"00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00","netmask":"ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff","first_host":"00:00:00:00:00:00:00:00:00:00:ff:ff:c0:a8:01:23","last_host":"00:00:00:00:00:00:00:00:00:00:ff:ff:c0:a8:01:23"},"bin":{"ip":"00000000000000000000000000000000000000000000000000000000000000000000000000000000111111111111111111000000101010000000000100100011","network":"00000000000000000000000000000000000000000000000000000000000000000000000000000000111111111111111111000000101010000000000100100011","broadcast":"00000000000000000000000000000000000000000000000000000000000000000000000000000000111111111111111111000000101010000000000100100011","hostmask":"00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000","netmask":"11111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111","first_host":"00000000000000000000000000000000000000000000000000000000000000000000000000000000111111111111111111000000101010000000000100100011","last_host":"00000000000000000000000000000000000000000000000000000000000000000000000000000000111111111111111111000000101010000000000100100011"}}''') + self.assertEqual(jc.parsers.ip_address.parse(data, quiet=True), expected) + + + def test_ip_address_ipv6_6to4(self): + """ + Test ipv6 6to4 address string + """ + data = r'2002:c000:204::/48' + expected = json.loads(r'''{"version":6,"max_prefix_length":128,"ip":"2002:c000:204::","ip_compressed":"2002:c000:204::","ip_exploded":"2002:c000:0204:0000:0000:0000:0000:0000","scope_id":null,"ipv4_mapped":null,"six_to_four":"192.0.2.4","teredo_client":null,"teredo_server":null,"dns_ptr":"0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.4.0.2.0.0.0.0.c.2.0.0.2.ip6.arpa","network":"2002:c000:204::","broadcast":"2002:c000:204:ffff:ffff:ffff:ffff:ffff","hostmask":"::ffff:ffff:ffff:ffff:ffff","netmask":"ffff:ffff:ffff::","cidr_netmask":48,"hosts":1208925819614629174706174,"first_host":"2002:c000:204::1","last_host":"2002:c000:204:ffff:ffff:ffff:ffff:fffe","is_multicast":false,"is_private":false,"is_global":true,"is_link_local":false,"is_loopback":false,"is_reserved":false,"is_unspecified":false,"int":{"ip":42549574682102084431821433448024768512,"network":42549574682102084431821433448024768512,"broadcast":42549574682103293357641048077199474687,"first_host":42549574682102084431821433448024768513,"last_host":42549574682103293357641048077199474686},"hex":{"ip":"20:02:c0:00:02:04:00:00:00:00:00:00:00:00:00:00","network":"20:02:c0:00:02:04:00:00:00:00:00:00:00:00:00:00","broadcast":"20:02:c0:00:02:04:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff","hostmask":"00:00:00:00:00:00:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff","netmask":"ff:ff:ff:ff:ff:ff:00:00:00:00:00:00:00:00:00:00","first_host":"20:02:c0:00:02:04:00:00:00:00:00:00:00:00:00:01","last_host":"20:02:c0:00:02:04:ff:ff:ff:ff:ff:ff:ff:ff:ff:fe"},"bin":{"ip":"00100000000000101100000000000000000000100000010000000000000000000000000000000000000000000000000000000000000000000000000000000000","network":"00100000000000101100000000000000000000100000010000000000000000000000000000000000000000000000000000000000000000000000000000000000","broadcast":"00100000000000101100000000000000000000100000010011111111111111111111111111111111111111111111111111111111111111111111111111111111","hostmask":"00000000000000000000000000000000000000000000000011111111111111111111111111111111111111111111111111111111111111111111111111111111","netmask":"11111111111111111111111111111111111111111111111100000000000000000000000000000000000000000000000000000000000000000000000000000000","first_host":"00100000000000101100000000000000000000100000010000000000000000000000000000000000000000000000000000000000000000000000000000000001","last_host":"00100000000000101100000000000000000000100000010011111111111111111111111111111111111111111111111111111111111111111111111111111110"}}''') + self.assertEqual(jc.parsers.ip_address.parse(data, quiet=True), expected) + + + def test_ip_address_ipv6_teredo(self): + """ + Test ipv6 teredo address string + """ + data = r'2001:0000:4136:e378:8000:63bf:3fff:fdd2' + expected = json.loads(r'''{"version":6,"max_prefix_length":128,"ip":"2001:0:4136:e378:8000:63bf:3fff:fdd2","ip_compressed":"2001:0:4136:e378:8000:63bf:3fff:fdd2","ip_exploded":"2001:0000:4136:e378:8000:63bf:3fff:fdd2","scope_id":null,"ipv4_mapped":null,"six_to_four":null,"teredo_client":"192.0.2.45","teredo_server":"65.54.227.120","dns_ptr":"2.d.d.f.f.f.f.3.f.b.3.6.0.0.0.8.8.7.3.e.6.3.1.4.0.0.0.0.1.0.0.2.ip6.arpa","network":"2001:0:4136:e378:8000:63bf:3fff:fdd2","broadcast":"2001:0:4136:e378:8000:63bf:3fff:fdd2","hostmask":"::","netmask":"ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff","cidr_netmask":128,"hosts":1,"first_host":"2001:0:4136:e378:8000:63bf:3fff:fdd2","last_host":"2001:0:4136:e378:8000:63bf:3fff:fdd2","is_multicast":false,"is_private":true,"is_global":false,"is_link_local":false,"is_loopback":false,"is_reserved":false,"is_unspecified":false,"int":{"ip":42540488182158724593221357832373272018,"network":42540488182158724593221357832373272018,"broadcast":42540488182158724593221357832373272018,"first_host":42540488182158724593221357832373272018,"last_host":42540488182158724593221357832373272018},"hex":{"ip":"20:01:00:00:41:36:e3:78:80:00:63:bf:3f:ff:fd:d2","network":"20:01:00:00:41:36:e3:78:80:00:63:bf:3f:ff:fd:d2","broadcast":"20:01:00:00:41:36:e3:78:80:00:63:bf:3f:ff:fd:d2","hostmask":"00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00","netmask":"ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff","first_host":"20:01:00:00:41:36:e3:78:80:00:63:bf:3f:ff:fd:d2","last_host":"20:01:00:00:41:36:e3:78:80:00:63:bf:3f:ff:fd:d2"},"bin":{"ip":"00100000000000010000000000000000010000010011011011100011011110001000000000000000011000111011111100111111111111111111110111010010","network":"00100000000000010000000000000000010000010011011011100011011110001000000000000000011000111011111100111111111111111111110111010010","broadcast":"00100000000000010000000000000000010000010011011011100011011110001000000000000000011000111011111100111111111111111111110111010010","hostmask":"00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000","netmask":"11111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111","first_host":"00100000000000010000000000000000010000010011011011100011011110001000000000000000011000111011111100111111111111111111110111010010","last_host":"00100000000000010000000000000000010000010011011011100011011110001000000000000000011000111011111100111111111111111111110111010010"}}''') + self.assertEqual(jc.parsers.ip_address.parse(data, quiet=True), expected) + + +if __name__ == '__main__': + unittest.main() From 4bb4cced90c47036a053e8b1f9be68e2ad4694d4 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 28 Jul 2022 10:25:21 -0700 Subject: [PATCH 017/116] fixes for python < 3.9 --- jc/parsers/ip_address.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/jc/parsers/ip_address.py b/jc/parsers/ip_address.py index 03b7c18ca..8f2bcf2a0 100644 --- a/jc/parsers/ip_address.py +++ b/jc/parsers/ip_address.py @@ -460,6 +460,7 @@ } """ from typing import Dict, Union +import re import binascii import ipaddress import jc.utils @@ -539,7 +540,12 @@ def parse( except Exception: data = data.strip() - interface = ipaddress.ip_interface(data) + # python versions < 3.9 do not handle the ipv6 scope, so remove it if parsing fails + try: + interface = ipaddress.ip_interface(data) + except ValueError: + data = re.sub(r'%[a-zA-Z0-9]*[^/]', data) + interface = ipaddress.ip_interface(data) network_string = str(interface.network).split('/')[0] network_cidr = int(str(interface.with_prefixlen).split('/')[1]) @@ -566,7 +572,12 @@ def parse( teredo_client = None teredo_server = None if interface.version == 6: - scope_id = interface.scope_id + # scope_id not available in python version < 3.9 + try: + scope_id = interface.scope_id + except AttributeError: + pass + ipv4_mapped = str(interface.ipv4_mapped) if interface.ipv4_mapped else None sixtofour = str(interface.sixtofour) if interface.sixtofour else None teredo_client = str(interface.teredo[1]) if interface.teredo else None From 4b245952fe7e31b7a05db02e1f8d4b00234092d0 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 28 Jul 2022 10:36:09 -0700 Subject: [PATCH 018/116] fix for ipv6 on python < 3.9 --- jc/parsers/ip_address.py | 2 +- tests/test_ip_address.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/jc/parsers/ip_address.py b/jc/parsers/ip_address.py index 8f2bcf2a0..330e68b0b 100644 --- a/jc/parsers/ip_address.py +++ b/jc/parsers/ip_address.py @@ -544,7 +544,7 @@ def parse( try: interface = ipaddress.ip_interface(data) except ValueError: - data = re.sub(r'%[a-zA-Z0-9]*[^/]', data) + data = re.sub(r'%[a-zA-Z0-9]*[^/]', '', data) interface = ipaddress.ip_interface(data) network_string = str(interface.network).split('/')[0] diff --git a/tests/test_ip_address.py b/tests/test_ip_address.py index 056cd9a7a..ca04bc258 100644 --- a/tests/test_ip_address.py +++ b/tests/test_ip_address.py @@ -61,8 +61,8 @@ def test_ip_address_ipv6_cidr_scope(self): """ Test CIDR ipv6 address with scope string """ - data = r'127:0:de::1%128/96' - expected = json.loads(r'''{"version":6,"max_prefix_length":128,"ip":"127:0:de::1","ip_compressed":"127:0:de::1","ip_exploded":"0127:0000:00de:0000:0000:0000:0000:0001","scope_id":"128","ipv4_mapped":null,"six_to_four":null,"teredo_client":null,"teredo_server":null,"dns_ptr":"1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.e.d.0.0.0.0.0.0.7.2.1.0.ip6.arpa","network":"127:0:de::","broadcast":"127:0:de::ffff:ffff","hostmask":"::ffff:ffff","netmask":"ffff:ffff:ffff:ffff:ffff:ffff::","cidr_netmask":96,"hosts":4294967294,"first_host":"127:0:de::1","last_host":"127:0:de::ffff:fffe","is_multicast":false,"is_private":false,"is_global":true,"is_link_local":false,"is_loopback":false,"is_reserved":true,"is_unspecified":false,"int":{"ip":1531727573536155682370944093904699393,"network":1531727573536155682370944093904699392,"broadcast":1531727573536155682370944098199666687,"first_host":1531727573536155682370944093904699393,"last_host":1531727573536155682370944098199666686},"hex":{"ip":"01:27:00:00:00:de:00:00:00:00:00:00:00:00:00:01","network":"01:27:00:00:00:de:00:00:00:00:00:00:00:00:00:00","broadcast":"01:27:00:00:00:de:00:00:00:00:00:00:ff:ff:ff:ff","hostmask":"00:00:00:00:00:00:00:00:00:00:00:00:ff:ff:ff:ff","netmask":"ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:00:00:00:00","first_host":"01:27:00:00:00:de:00:00:00:00:00:00:00:00:00:01","last_host":"01:27:00:00:00:de:00:00:00:00:00:00:ff:ff:ff:fe"},"bin":{"ip":"00000001001001110000000000000000000000001101111000000000000000000000000000000000000000000000000000000000000000000000000000000001","network":"00000001001001110000000000000000000000001101111000000000000000000000000000000000000000000000000000000000000000000000000000000000","broadcast":"00000001001001110000000000000000000000001101111000000000000000000000000000000000000000000000000011111111111111111111111111111111","hostmask":"00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000011111111111111111111111111111111","netmask":"11111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100000000000000000000000000000000","first_host":"00000001001001110000000000000000000000001101111000000000000000000000000000000000000000000000000000000000000000000000000000000001","last_host":"00000001001001110000000000000000000000001101111000000000000000000000000000000000000000000000000011111111111111111111111111111110"}}''') + data = r'127:0:de::1%128aBc123/96' + expected = json.loads(r'''{"version":6,"max_prefix_length":128,"ip":"127:0:de::1","ip_compressed":"127:0:de::1","ip_exploded":"0127:0000:00de:0000:0000:0000:0000:0001","scope_id":"128aBc123","ipv4_mapped":null,"six_to_four":null,"teredo_client":null,"teredo_server":null,"dns_ptr":"1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.e.d.0.0.0.0.0.0.7.2.1.0.ip6.arpa","network":"127:0:de::","broadcast":"127:0:de::ffff:ffff","hostmask":"::ffff:ffff","netmask":"ffff:ffff:ffff:ffff:ffff:ffff::","cidr_netmask":96,"hosts":4294967294,"first_host":"127:0:de::1","last_host":"127:0:de::ffff:fffe","is_multicast":false,"is_private":false,"is_global":true,"is_link_local":false,"is_loopback":false,"is_reserved":true,"is_unspecified":false,"int":{"ip":1531727573536155682370944093904699393,"network":1531727573536155682370944093904699392,"broadcast":1531727573536155682370944098199666687,"first_host":1531727573536155682370944093904699393,"last_host":1531727573536155682370944098199666686},"hex":{"ip":"01:27:00:00:00:de:00:00:00:00:00:00:00:00:00:01","network":"01:27:00:00:00:de:00:00:00:00:00:00:00:00:00:00","broadcast":"01:27:00:00:00:de:00:00:00:00:00:00:ff:ff:ff:ff","hostmask":"00:00:00:00:00:00:00:00:00:00:00:00:ff:ff:ff:ff","netmask":"ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:ff:00:00:00:00","first_host":"01:27:00:00:00:de:00:00:00:00:00:00:00:00:00:01","last_host":"01:27:00:00:00:de:00:00:00:00:00:00:ff:ff:ff:fe"},"bin":{"ip":"00000001001001110000000000000000000000001101111000000000000000000000000000000000000000000000000000000000000000000000000000000001","network":"00000001001001110000000000000000000000001101111000000000000000000000000000000000000000000000000000000000000000000000000000000000","broadcast":"00000001001001110000000000000000000000001101111000000000000000000000000000000000000000000000000011111111111111111111111111111111","hostmask":"00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000011111111111111111111111111111111","netmask":"11111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100000000000000000000000000000000","first_host":"00000001001001110000000000000000000000001101111000000000000000000000000000000000000000000000000000000000000000000000000000000001","last_host":"00000001001001110000000000000000000000001101111000000000000000000000000000000000000000000000000011111111111111111111111111111110"}}''') self.assertEqual(jc.parsers.ip_address.parse(data, quiet=True), expected) From 1fcf844e8d2725189167f327f2c520167d01a65f Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 28 Jul 2022 10:50:03 -0700 Subject: [PATCH 019/116] manually pull scope_id for older python versions --- jc/parsers/ip_address.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/jc/parsers/ip_address.py b/jc/parsers/ip_address.py index 330e68b0b..6340735de 100644 --- a/jc/parsers/ip_address.py +++ b/jc/parsers/ip_address.py @@ -534,17 +534,25 @@ def parse( if jc.utils.has_data(data): + SCOPE_PATTERN = re.compile(r'%[a-zA-Z0-9]*[^/]') + # Accept IPs entered as integer notation try: data = int(data) # type: ignore except Exception: data = data.strip() - # python versions < 3.9 do not handle the ipv6 scope, so remove it if parsing fails + # python versions < 3.9 do not handle the ipv6 scope, so pop it + # and use instead of ipaddress.scope_id if parsing fails + scope_string = None try: interface = ipaddress.ip_interface(data) except ValueError: - data = re.sub(r'%[a-zA-Z0-9]*[^/]', '', data) + scope_match = re.match(SCOPE_PATTERN, data) + if scope_match: + scope_string = scope_match.group(0) + + data = re.sub(SCOPE_PATTERN, '', data) interface = ipaddress.ip_interface(data) network_string = str(interface.network).split('/')[0] @@ -576,7 +584,7 @@ def parse( try: scope_id = interface.scope_id except AttributeError: - pass + scope_id = scope_string ipv4_mapped = str(interface.ipv4_mapped) if interface.ipv4_mapped else None sixtofour = str(interface.sixtofour) if interface.sixtofour else None From 24960cd02beb4f6fbb8079bb9f528cf93ab9ef1a Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 28 Jul 2022 15:42:25 -0700 Subject: [PATCH 020/116] fix scope matching issue --- jc/parsers/ip_address.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/jc/parsers/ip_address.py b/jc/parsers/ip_address.py index 6340735de..6a7fcf586 100644 --- a/jc/parsers/ip_address.py +++ b/jc/parsers/ip_address.py @@ -545,12 +545,14 @@ def parse( # python versions < 3.9 do not handle the ipv6 scope, so pop it # and use instead of ipaddress.scope_id if parsing fails scope_string = None + try: interface = ipaddress.ip_interface(data) + except ValueError: - scope_match = re.match(SCOPE_PATTERN, data) + scope_match = re.search(SCOPE_PATTERN, data) if scope_match: - scope_string = scope_match.group(0) + scope_string = scope_match.group(0)[1:] data = re.sub(SCOPE_PATTERN, '', data) interface = ipaddress.ip_interface(data) From babb08c2e0baac24db809579075129e8f01e7391 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 28 Jul 2022 16:30:13 -0700 Subject: [PATCH 021/116] formatting --- jc/parsers/ip_address.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jc/parsers/ip_address.py b/jc/parsers/ip_address.py index 6a7fcf586..e470d08e5 100644 --- a/jc/parsers/ip_address.py +++ b/jc/parsers/ip_address.py @@ -663,8 +663,8 @@ def parse( }, 'bin': { 'ip': _bin_format(bare_ip, interface.max_prefixlen), - 'network': _bin_format(network_ipobj, interface.max_prefixlen), - 'broadcast': _bin_format(broadcast_ipobj, interface.max_prefixlen), + 'network': _bin_format(network_ipobj, interface.max_prefixlen), + 'broadcast': _bin_format(broadcast_ipobj, interface.max_prefixlen), 'hostmask': _bin_format(hostmask_ipobj, interface.max_prefixlen), 'netmask': _bin_format(netmask_ipobj, interface.max_prefixlen), 'first_host': _bin_format(first_host_ipobj, interface.max_prefixlen), From 7f73740fc7cacbba2d9a13e3249047c21dee2f2e Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 28 Jul 2022 17:18:01 -0700 Subject: [PATCH 022/116] add dotted netmask test --- tests/test_ip_address.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/test_ip_address.py b/tests/test_ip_address.py index ca04bc258..e2e1fafe1 100644 --- a/tests/test_ip_address.py +++ b/tests/test_ip_address.py @@ -30,6 +30,15 @@ def test_ip_address_ipv4_cidr(self): self.assertEqual(jc.parsers.ip_address.parse(data, quiet=True), expected) + def test_ip_address_ipv4_dotnetmask(self): + """ + Test ipv4 address with a dotted netmask string + """ + data = r'192.168.0.1/255.255.128.0' + expected = json.loads(r'''{"version":4,"max_prefix_length":32,"ip":"192.168.0.1","ip_compressed":"192.168.0.1","ip_exploded":"192.168.0.1","scope_id":null,"ipv4_mapped":null,"six_to_four":null,"teredo_client":null,"teredo_server":null,"dns_ptr":"1.0.168.192.in-addr.arpa","network":"192.168.0.0","broadcast":"192.168.127.255","hostmask":"0.0.127.255","netmask":"255.255.128.0","cidr_netmask":17,"hosts":32766,"first_host":"192.168.0.1","last_host":"192.168.127.254","is_multicast":false,"is_private":true,"is_global":false,"is_link_local":false,"is_loopback":false,"is_reserved":false,"is_unspecified":false,"int":{"ip":3232235521,"network":3232235520,"broadcast":3232268287,"first_host":3232235521,"last_host":3232268286},"hex":{"ip":"c0:a8:00:01","network":"c0:a8:00:00","broadcast":"c0:a8:7f:ff","hostmask":"00:00:7f:ff","netmask":"ff:ff:80:00","first_host":"c0:a8:00:01","last_host":"c0:a8:7f:fe"},"bin":{"ip":"11000000101010000000000000000001","network":"11000000101010000000000000000000","broadcast":"11000000101010000111111111111111","hostmask":"00000000000000000111111111111111","netmask":"11111111111111111000000000000000","first_host":"11000000101010000000000000000001","last_host":"11000000101010000111111111111110"}}''') + self.assertEqual(jc.parsers.ip_address.parse(data, quiet=True), expected) + + def test_ip_address_ipv4_integer(self): """ Test ipv4 address integer string From 790cdf299629503265486596fa54c4cc8fca3a59 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 29 Jul 2022 13:22:16 -0700 Subject: [PATCH 023/116] add device_qualifier section --- jc/parsers/lsusb.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/jc/parsers/lsusb.py b/jc/parsers/lsusb.py index be1751472..701d05446 100644 --- a/jc/parsers/lsusb.py +++ b/jc/parsers/lsusb.py @@ -329,6 +329,7 @@ def __init__(self): self.report_descriptors_list = [] self.hub_descriptor_list = [] self.hub_port_status_list = [] + self.device_qualifier_list = [] self.device_status_list = [] @staticmethod @@ -484,7 +485,8 @@ def _set_sections(self, line): ' HID Device Descriptor:': 'hid_device_descriptor', ' Report Descriptors:': 'report_descriptors', 'Hub Descriptor:': 'hub_descriptor', - ' Hub Port Status:': 'hub_port_status' + ' Hub Port Status:': 'hub_port_status', + 'Device Qualifier (for other device speed):': 'device_qualifier' } for sec_string, section_val in string_section_map.items(): @@ -508,7 +510,8 @@ def _populate_lists(self, line): 'hid_device_descriptor': self.hid_device_descriptor_list, 'report_descriptors': self.report_descriptors_list, 'endpoint_descriptor': self.endpoint_descriptor_list, - 'hub_descriptor': self.hub_descriptor_list + 'hub_descriptor': self.hub_descriptor_list, + 'device_qualifier': self.device_qualifier_list } for sec in section_list_map: @@ -545,6 +548,7 @@ def _populate_schema(self): ['device_descriptor']['configuration_descriptor']['interface_descriptors'][0]['endpoint_descriptors'][0] = {} ['hub_descriptor'] = {} ['hub_descriptor']['hub_port_status'] = {} + ['device_qualifier'] = {} ['device_status'] = {} """ for idx, item in enumerate(self.bus_list): @@ -796,6 +800,12 @@ def _populate_schema(self): self.output_line['hub_descriptor']['hub_port_status'].update(hps) del self.output_line['hub_descriptor']['hub_port_status'][keyname]['_state'] + for dq in self.device_qualifier_list: + keyname = tuple(dq.keys())[0] + if '_state' in dq[keyname] and dq[keyname]['_state']['bus_idx'] == idx: + self.output_line['device_qualifier'].update(dq) + del self.output_line['device_qualifier'][keyname]['_state'] + for ds in self.device_status_list: if '_state' in ds and ds['_state']['bus_idx'] == idx: self.output_line['device_status'].update(ds) From c27591e83856c66bd8ad011240e76ad8945fa102 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 29 Jul 2022 13:26:07 -0700 Subject: [PATCH 024/116] update schema docs --- jc/parsers/lsusb.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/jc/parsers/lsusb.py b/jc/parsers/lsusb.py index 701d05446..a97dea04b 100644 --- a/jc/parsers/lsusb.py +++ b/jc/parsers/lsusb.py @@ -129,6 +129,12 @@ } } }, + "device_qualifier": { + "": { + "value": string, + "description": string + } + }, "device_status": { "value": string, "description": string From f22dd3aa88aa6177cdfd6eebbc1b20d62b5f172f Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 29 Jul 2022 13:26:51 -0700 Subject: [PATCH 025/116] doc update --- docs/parsers/lsusb.md | 6 ++++++ man/jc.1 | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/parsers/lsusb.md b/docs/parsers/lsusb.md index 84033124b..0de839cbd 100644 --- a/docs/parsers/lsusb.md +++ b/docs/parsers/lsusb.md @@ -134,6 +134,12 @@ Schema: } } }, + "device_qualifier": { + "": { + "value": string, + "description": string + } + }, "device_status": { "value": string, "description": string diff --git a/man/jc.1 b/man/jc.1 index 5b2a46107..485123e51 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -1,4 +1,4 @@ -.TH jc 1 2022-07-28 1.20.5 "JSON Convert" +.TH jc 1 2022-07-29 1.20.5 "JSON Convert" .SH NAME \fBjc\fP \- JSON Convert JSONifies the output of many CLI tools and file-types .SH SYNOPSIS From 37b830a2fd14d3f05a6fb890a725819907a21a3c Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 29 Jul 2022 13:35:45 -0700 Subject: [PATCH 026/116] doc update --- CHANGELOG | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG b/CHANGELOG index 81818ca83..b679f8102 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -2,6 +2,7 @@ jc changelog xxxxxxxx v1.20.5 - Add IP Address string parser +- Fix `lsusb` command parser for output containing a `Device Qualifier` section - Change LANG=C to LC_ALL=C in locale instructions 20220723 v1.20.4 From 4075bce26af80100b819408d3bf9ffc36c46deb1 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 29 Jul 2022 13:38:21 -0700 Subject: [PATCH 027/116] version bump --- docs/parsers/lsusb.md | 2 +- jc/parsers/lsusb.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/parsers/lsusb.md b/docs/parsers/lsusb.md index 0de839cbd..c6d067a61 100644 --- a/docs/parsers/lsusb.md +++ b/docs/parsers/lsusb.md @@ -290,4 +290,4 @@ Returns: ### Parser Information Compatibility: linux -Version 1.1 by Kelly Brazil (kellyjonbrazil@gmail.com) +Version 1.2 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/jc/parsers/lsusb.py b/jc/parsers/lsusb.py index a97dea04b..c61400ac9 100644 --- a/jc/parsers/lsusb.py +++ b/jc/parsers/lsusb.py @@ -269,7 +269,7 @@ class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.1' + version = '1.2' description = '`lsusb` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' From ebec5c98b845223ae3cc374da219468f97328d7a Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sat, 30 Jul 2022 10:04:52 -0700 Subject: [PATCH 028/116] handle Binary Object Store Descriptor section (not implemented) --- jc/parsers/lsusb.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jc/parsers/lsusb.py b/jc/parsers/lsusb.py index c61400ac9..58bd6072c 100644 --- a/jc/parsers/lsusb.py +++ b/jc/parsers/lsusb.py @@ -492,7 +492,8 @@ def _set_sections(self, line): ' Report Descriptors:': 'report_descriptors', 'Hub Descriptor:': 'hub_descriptor', ' Hub Port Status:': 'hub_port_status', - 'Device Qualifier (for other device speed):': 'device_qualifier' + 'Device Qualifier (for other device speed):': 'device_qualifier', + 'Binary Object Store Descriptor:': None # not implemented } for sec_string, section_val in string_section_map.items(): From 0993fec114343fadd0e3b5fb2df7bb5c991a1fd4 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sat, 30 Jul 2022 11:10:10 -0700 Subject: [PATCH 029/116] add tests --- .../generic/lsusb-binary-object-store.json | 1 + .../generic/lsusb-binary-object-store.out | 1373 +++++++++++++++++ .../generic/lsusb-device-qualifier.json | 1 + tests/test_lsusb.py | 24 + 4 files changed, 1399 insertions(+) create mode 100644 tests/fixtures/generic/lsusb-binary-object-store.json create mode 100644 tests/fixtures/generic/lsusb-binary-object-store.out create mode 100644 tests/fixtures/generic/lsusb-device-qualifier.json diff --git a/tests/fixtures/generic/lsusb-binary-object-store.json b/tests/fixtures/generic/lsusb-binary-object-store.json new file mode 100644 index 000000000..5c66047da --- /dev/null +++ b/tests/fixtures/generic/lsusb-binary-object-store.json @@ -0,0 +1 @@ +[{"bus":"002","device":"001","id":"1d6b:0003","description":"Linux Foundation 3.0 root hub","device_descriptor":{"bLength":{"value":"18"},"bDescriptorType":{"value":"1"},"bcdUSB":{"value":"3.10"},"bDeviceClass":{"value":"9","description":"Hub"},"bDeviceSubClass":{"value":"0"},"bDeviceProtocol":{"value":"3"},"bMaxPacketSize0":{"value":"9"},"idVendor":{"value":"0x1d6b","description":"Linux Foundation"},"idProduct":{"value":"0x0003","description":"3.0 root hub"},"bcdDevice":{"value":"4.18"},"iManufacturer":{"value":"3","description":"Linux 4.18.0-372.9.1.el8.x86_64 xhci-hcd"},"iProduct":{"value":"2","description":"xHCI Host Controller"},"iSerial":{"value":"1","description":"0000:02:00.0"},"bNumConfigurations":{"value":"1"},"configuration_descriptor":{"bLength":{"value":"9"},"bDescriptorType":{"value":"2"},"wTotalLength":{"value":"0x001f"},"bNumInterfaces":{"value":"1"},"bConfigurationValue":{"value":"1"},"iConfiguration":{"value":"0"},"bmAttributes":{"value":"0xe0","attributes":["Self Powered","Remote Wakeup"]},"MaxPower":{"description":"0mA"},"interface_descriptors":[{"bLength":{"value":"9"},"bDescriptorType":{"value":"4"},"bInterfaceNumber":{"value":"0"},"bAlternateSetting":{"value":"0"},"bNumEndpoints":{"value":"1"},"bInterfaceClass":{"value":"9","description":"Hub"},"bInterfaceSubClass":{"value":"0"},"bInterfaceProtocol":{"value":"0","description":"Full speed (or root) hub"},"iInterface":{"value":"0"},"endpoint_descriptors":[{"bLength":{"value":"7"},"bDescriptorType":{"value":"5"},"bEndpointAddress":{"value":"0x81","description":"EP 1 IN"},"bmAttributes":{"value":"3","attributes":["Transfer Type Interrupt","Synch Type None","Usage Type Data"]},"wMaxPacketSize":{"value":"0x0004","description":"1x 4 bytes"},"bInterval":{"value":"12"},"bMaxBurst":{"value":"0"}}]}]}},"hub_descriptor":{"bLength":{"value":"12"},"bDescriptorType":{"value":"42"},"nNbrPorts":{"value":"4"},"wHubCharacteristic":{"value":"0x000a","attributes":["No power switching (usb 1.0)","Per-port overcurrent protection"]},"bPwrOn2PwrGood":{"value":"50 *","description":"2 milli seconds"},"bHubContrCurrent":{"value":"0","description":"milli Ampere"},"bHubDecLat":{"value":"0.0","description":"micro seconds"},"wHubDelay":{"value":"0","description":"nano seconds"},"DeviceRemovable":{"value":"0x00"},"hub_port_status":{"Port 1":{"value":"0000.02a0","attributes":["lowspeed","L1"]},"Port 2":{"value":"0000.02a0","attributes":["lowspeed","L1"]},"Port 3":{"value":"0000.02a0","attributes":["lowspeed","L1"]},"Port 4":{"value":"0000.02a0","attributes":["lowspeed","L1"]}}},"device_status":{"value":"0x0001","description":"Self Powered"}},{"bus":"003","device":"002","id":"2109:2813","description":"VIA Labs, Inc. VL813 Hub","device_descriptor":{"bLength":{"value":"18"},"bDescriptorType":{"value":"1"},"bcdUSB":{"value":"2.10"},"bDeviceClass":{"value":"9","description":"Hub"},"bDeviceSubClass":{"value":"0"},"bDeviceProtocol":{"value":"1","description":"Single TT"},"bMaxPacketSize0":{"value":"64"},"idVendor":{"value":"0x2109","description":"VIA Labs, Inc."},"idProduct":{"value":"0x2813","description":"VL813 Hub"},"bcdDevice":{"value":"90.11"},"iManufacturer":{"value":"1","description":"VIA Labs, Inc."},"iProduct":{"value":"2","description":"USB2.0 Hub"},"iSerial":{"value":"0"},"bNumConfigurations":{"value":"1"},"configuration_descriptor":{"bLength":{"value":"9"},"bDescriptorType":{"value":"2"},"wTotalLength":{"value":"0x0019"},"bNumInterfaces":{"value":"1"},"bConfigurationValue":{"value":"1"},"iConfiguration":{"value":"0"},"bmAttributes":{"value":"0xe0","attributes":["Self Powered","Remote Wakeup"]},"MaxPower":{"description":"0mA"},"interface_descriptors":[{"bLength":{"value":"9"},"bDescriptorType":{"value":"4"},"bInterfaceNumber":{"value":"0"},"bAlternateSetting":{"value":"0"},"bNumEndpoints":{"value":"1"},"bInterfaceClass":{"value":"9","description":"Hub"},"bInterfaceSubClass":{"value":"0"},"bInterfaceProtocol":{"value":"0","description":"Full speed (or root) hub"},"iInterface":{"value":"0"},"endpoint_descriptors":[{"bLength":{"value":"7"},"bDescriptorType":{"value":"5"},"bEndpointAddress":{"value":"0x81","description":"EP 1 IN"},"bmAttributes":{"value":"3","attributes":["Transfer Type Interrupt","Synch Type None","Usage Type Data"]},"wMaxPacketSize":{"value":"0x0001","description":"1x 1 bytes"},"bInterval":{"value":"12"}}]}]}},"hub_descriptor":{"bLength":{"value":"9"},"bDescriptorType":{"value":"41"},"nNbrPorts":{"value":"4"},"wHubCharacteristic":{"value":"0x00e9","attributes":["Per-port power switching","Per-port overcurrent protection","TT think time 32 FS bits","Port indicators"]},"bPwrOn2PwrGood":{"value":"50 *","description":"2 milli seconds"},"bHubContrCurrent":{"value":"100","description":"milli Ampere"},"DeviceRemovable":{"value":"0x00"},"PortPwrCtrlMask":{"value":"0xff"},"hub_port_status":{"Port 1":{"value":"0000.0503","attributes":["highspeed","power","enable","connect"]},"Port 2":{"value":"0000.0100","attributes":["power"]},"Port 3":{"value":"0000.0100","attributes":["power"]},"Port 4":{"value":"0000.0100","attributes":["power"]}}},"device_status":{"value":"0x0001","description":"Self Powered"}},{"bus":"003","device":"003","id":"2109:2813","description":"VIA Labs, Inc. VL813 Hub","device_descriptor":{"bLength":{"value":"18"},"bDescriptorType":{"value":"1"},"bcdUSB":{"value":"2.10"},"bDeviceClass":{"value":"9","description":"Hub"},"bDeviceSubClass":{"value":"0"},"bDeviceProtocol":{"value":"1","description":"Single TT"},"bMaxPacketSize0":{"value":"64"},"idVendor":{"value":"0x2109","description":"VIA Labs, Inc."},"idProduct":{"value":"0x2813","description":"VL813 Hub"},"bcdDevice":{"value":"90.11"},"iManufacturer":{"value":"1","description":"VIA Labs, Inc."},"iProduct":{"value":"2","description":"USB2.0 Hub"},"iSerial":{"value":"0"},"bNumConfigurations":{"value":"1"},"configuration_descriptor":{"bLength":{"value":"9"},"bDescriptorType":{"value":"2"},"wTotalLength":{"value":"0x0019"},"bNumInterfaces":{"value":"1"},"bConfigurationValue":{"value":"1"},"iConfiguration":{"value":"0"},"bmAttributes":{"value":"0xe0","attributes":["Self Powered","Remote Wakeup"]},"MaxPower":{"description":"0mA"},"interface_descriptors":[{"bLength":{"value":"9"},"bDescriptorType":{"value":"4"},"bInterfaceNumber":{"value":"0"},"bAlternateSetting":{"value":"0"},"bNumEndpoints":{"value":"1"},"bInterfaceClass":{"value":"9","description":"Hub"},"bInterfaceSubClass":{"value":"0"},"bInterfaceProtocol":{"value":"0","description":"Full speed (or root) hub"},"iInterface":{"value":"0"},"endpoint_descriptors":[{"bLength":{"value":"7"},"bDescriptorType":{"value":"5"},"bEndpointAddress":{"value":"0x81","description":"EP 1 IN"},"bmAttributes":{"value":"3","attributes":["Transfer Type Interrupt","Synch Type None","Usage Type Data"]},"wMaxPacketSize":{"value":"0x0001","description":"1x 1 bytes"},"bInterval":{"value":"12"}}]}]}},"hub_descriptor":{"bLength":{"value":"9"},"bDescriptorType":{"value":"41"},"nNbrPorts":{"value":"4"},"wHubCharacteristic":{"value":"0x00e9","attributes":["Per-port power switching","Per-port overcurrent protection","TT think time 32 FS bits","Port indicators"]},"bPwrOn2PwrGood":{"value":"50 *","description":"2 milli seconds"},"bHubContrCurrent":{"value":"100","description":"milli Ampere"},"DeviceRemovable":{"value":"0x00"},"PortPwrCtrlMask":{"value":"0xff"},"hub_port_status":{"Port 1":{"value":"0000.0503","attributes":["highspeed","power","enable","connect"]},"Port 2":{"value":"0000.0100","attributes":["power"]},"Port 3":{"value":"0000.0100","attributes":["power"]},"Port 4":{"value":"0000.0100","attributes":["power"]}}},"device_status":{"value":"0x0001","description":"Self Powered"}},{"bus":"004","device":"001","id":"1d6b:0003","description":"Linux Foundation 3.0 root hub","device_descriptor":{"bLength":{"value":"18"},"bDescriptorType":{"value":"1"},"bcdUSB":{"value":"3.00"},"bDeviceClass":{"value":"9","description":"Hub"},"bDeviceSubClass":{"value":"0"},"bDeviceProtocol":{"value":"3"},"bMaxPacketSize0":{"value":"9"},"idVendor":{"value":"0x1d6b","description":"Linux Foundation"},"idProduct":{"value":"0x0003","description":"3.0 root hub"},"bcdDevice":{"value":"4.18"},"iManufacturer":{"value":"3","description":"Linux 4.18.0-372.9.1.el8.x86_64 xhci-hcd"},"iProduct":{"value":"2","description":"xHCI Host Controller"},"iSerial":{"value":"1","description":"0000:05:00.0"},"bNumConfigurations":{"value":"1"},"configuration_descriptor":{"bLength":{"value":"9"},"bDescriptorType":{"value":"2"},"wTotalLength":{"value":"0x001f"},"bNumInterfaces":{"value":"1"},"bConfigurationValue":{"value":"1"},"iConfiguration":{"value":"0"},"bmAttributes":{"value":"0xe0","attributes":["Self Powered","Remote Wakeup"]},"MaxPower":{"description":"0mA"},"interface_descriptors":[{"bLength":{"value":"9"},"bDescriptorType":{"value":"4"},"bInterfaceNumber":{"value":"0"},"bAlternateSetting":{"value":"0"},"bNumEndpoints":{"value":"1"},"bInterfaceClass":{"value":"9","description":"Hub"},"bInterfaceSubClass":{"value":"0"},"bInterfaceProtocol":{"value":"0","description":"Full speed (or root) hub"},"iInterface":{"value":"0"},"endpoint_descriptors":[{"bLength":{"value":"7"},"bDescriptorType":{"value":"5"},"bEndpointAddress":{"value":"0x81","description":"EP 1 IN"},"bmAttributes":{"value":"3","attributes":["Transfer Type Interrupt","Synch Type None","Usage Type Data"]},"wMaxPacketSize":{"value":"0x0004","description":"1x 4 bytes"},"bInterval":{"value":"12"},"bMaxBurst":{"value":"0"}}]}]}},"hub_descriptor":{"bLength":{"value":"12"},"bDescriptorType":{"value":"42"},"nNbrPorts":{"value":"2"},"wHubCharacteristic":{"value":"0x0009","attributes":["Per-port power switching","Per-port overcurrent protection"]},"bPwrOn2PwrGood":{"value":"50 *","description":"2 milli seconds"},"bHubContrCurrent":{"value":"0","description":"milli Ampere"},"bHubDecLat":{"value":"0.0","description":"micro seconds"},"wHubDelay":{"value":"0","description":"nano seconds"},"DeviceRemovable":{"value":"0x00"},"hub_port_status":{"Port 1":{"value":"0000.0263","attributes":["5Gbps","power","suspend","enable","connect"]},"Port 2":{"value":"0000.02a0","attributes":["5Gbps","power","Rx.Detect"]}}},"device_status":{"value":"0x0001","description":"Self Powered"}},{"bus":"004","device":"002","id":"2109:0813","description":"VIA Labs, Inc. VL813 Hub","device_descriptor":{"bLength":{"value":"18"},"bDescriptorType":{"value":"1"},"bcdUSB":{"value":"3.00"},"bDeviceClass":{"value":"9","description":"Hub"},"bDeviceSubClass":{"value":"0"},"bDeviceProtocol":{"value":"3"},"bMaxPacketSize0":{"value":"9"},"idVendor":{"value":"0x2109","description":"VIA Labs, Inc."},"idProduct":{"value":"0x0813","description":"VL813 Hub"},"bcdDevice":{"value":"90.11"},"iManufacturer":{"value":"1","description":"VIA Labs, Inc."},"iProduct":{"value":"2","description":"USB3.0 Hub"},"iSerial":{"value":"0"},"bNumConfigurations":{"value":"1"},"configuration_descriptor":{"bLength":{"value":"9"},"bDescriptorType":{"value":"2"},"wTotalLength":{"value":"0x001f"},"bNumInterfaces":{"value":"1"},"bConfigurationValue":{"value":"1"},"iConfiguration":{"value":"0"},"bmAttributes":{"value":"0xe0","attributes":["Self Powered","Remote Wakeup"]},"MaxPower":{"description":"0mA"},"interface_descriptors":[{"bLength":{"value":"9"},"bDescriptorType":{"value":"4"},"bInterfaceNumber":{"value":"0"},"bAlternateSetting":{"value":"0"},"bNumEndpoints":{"value":"1"},"bInterfaceClass":{"value":"9","description":"Hub"},"bInterfaceSubClass":{"value":"0"},"bInterfaceProtocol":{"value":"0","description":"Full speed (or root) hub"},"iInterface":{"value":"0"},"endpoint_descriptors":[{"bLength":{"value":"7"},"bDescriptorType":{"value":"5"},"bEndpointAddress":{"value":"0x81","description":"EP 1 IN"},"bmAttributes":{"value":"19","attributes":["Transfer Type Interrupt","Synch Type None","Usage Type Feedback"]},"wMaxPacketSize":{"value":"0x0002","description":"1x 2 bytes"},"bInterval":{"value":"8"},"bMaxBurst":{"value":"0"}}]}]}},"hub_descriptor":{"bLength":{"value":"12"},"bDescriptorType":{"value":"42"},"nNbrPorts":{"value":"4"},"wHubCharacteristic":{"value":"0x0009","attributes":["Per-port power switching","Per-port overcurrent protection"]},"bPwrOn2PwrGood":{"value":"100 *","description":"2 milli seconds"},"bHubContrCurrent":{"value":"0","description":"milli Ampere"},"bHubDecLat":{"value":"0.4","description":"micro seconds"},"wHubDelay":{"value":"4004","description":"nano seconds"},"DeviceRemovable":{"value":"0x00"},"hub_port_status":{"Port 1":{"value":"0000.0263","attributes":["5Gbps","power","suspend","enable","connect"]},"Port 2":{"value":"0000.02a0","attributes":["5Gbps","power","Rx.Detect"]},"Port 3":{"value":"0000.02a0","attributes":["5Gbps","power","Rx.Detect"]},"Port 4":{"value":"0000.02a0","attributes":["5Gbps","power","Rx.Detect"]}}},"device_status":{"value":"0x0001","description":"Self Powered"}},{"bus":"004","device":"003","id":"2109:0813","description":"VIA Labs, Inc. VL813 Hub","device_descriptor":{"bLength":{"value":"18"},"bDescriptorType":{"value":"1"},"bcdUSB":{"value":"3.00"},"bDeviceClass":{"value":"9","description":"Hub"},"bDeviceSubClass":{"value":"0"},"bDeviceProtocol":{"value":"3"},"bMaxPacketSize0":{"value":"9"},"idVendor":{"value":"0x2109","description":"VIA Labs, Inc."},"idProduct":{"value":"0x0813","description":"VL813 Hub"},"bcdDevice":{"value":"90.11"},"iManufacturer":{"value":"1","description":"VIA Labs, Inc."},"iProduct":{"value":"2","description":"USB3.0 Hub"},"iSerial":{"value":"0"},"bNumConfigurations":{"value":"1"},"configuration_descriptor":{"bLength":{"value":"9"},"bDescriptorType":{"value":"2"},"wTotalLength":{"value":"0x001f"},"bNumInterfaces":{"value":"1"},"bConfigurationValue":{"value":"1"},"iConfiguration":{"value":"0"},"bmAttributes":{"value":"0xe0","attributes":["Self Powered","Remote Wakeup"]},"MaxPower":{"description":"0mA"},"interface_descriptors":[{"bLength":{"value":"9"},"bDescriptorType":{"value":"4"},"bInterfaceNumber":{"value":"0"},"bAlternateSetting":{"value":"0"},"bNumEndpoints":{"value":"1"},"bInterfaceClass":{"value":"9","description":"Hub"},"bInterfaceSubClass":{"value":"0"},"bInterfaceProtocol":{"value":"0","description":"Full speed (or root) hub"},"iInterface":{"value":"0"},"endpoint_descriptors":[{"bLength":{"value":"7"},"bDescriptorType":{"value":"5"},"bEndpointAddress":{"value":"0x81","description":"EP 1 IN"},"bmAttributes":{"value":"19","attributes":["Transfer Type Interrupt","Synch Type None","Usage Type Feedback"]},"wMaxPacketSize":{"value":"0x0002","description":"1x 2 bytes"},"bInterval":{"value":"8"},"bMaxBurst":{"value":"0"}}]}]}},"hub_descriptor":{"bLength":{"value":"12"},"bDescriptorType":{"value":"42"},"nNbrPorts":{"value":"4"},"wHubCharacteristic":{"value":"0x0009","attributes":["Per-port power switching","Per-port overcurrent protection"]},"bPwrOn2PwrGood":{"value":"100 *","description":"2 milli seconds"},"bHubContrCurrent":{"value":"0","description":"milli Ampere"},"bHubDecLat":{"value":"0.4","description":"micro seconds"},"wHubDelay":{"value":"4004","description":"nano seconds"},"DeviceRemovable":{"value":"0x00"},"hub_port_status":{"Port 1":{"value":"0000.02a0","attributes":["5Gbps","power","Rx.Detect"]},"Port 2":{"value":"0000.02a0","attributes":["5Gbps","power","Rx.Detect"]},"Port 3":{"value":"0000.02a0","attributes":["5Gbps","power","Rx.Detect"]},"Port 4":{"value":"0000.02a0","attributes":["5Gbps","power","Rx.Detect"]}}},"device_status":{"value":"0x0001","description":"Self Powered"}},{"bus":"006","device":"001","id":"1d6b:0003","description":"Linux Foundation 3.0 root hub","device_descriptor":{"bLength":{"value":"18"},"bDescriptorType":{"value":"1"},"bcdUSB":{"value":"3.10"},"bDeviceClass":{"value":"9","description":"Hub"},"bDeviceSubClass":{"value":"0"},"bDeviceProtocol":{"value":"3"},"bMaxPacketSize0":{"value":"9"},"idVendor":{"value":"0x1d6b","description":"Linux Foundation"},"idProduct":{"value":"0x0003","description":"3.0 root hub"},"bcdDevice":{"value":"4.18"},"iManufacturer":{"value":"3","description":"Linux 4.18.0-372.9.1.el8.x86_64 xhci-hcd"},"iProduct":{"value":"2","description":"xHCI Host Controller"},"iSerial":{"value":"1","description":"0000:0e:00.3"},"bNumConfigurations":{"value":"1"},"configuration_descriptor":{"bLength":{"value":"9"},"bDescriptorType":{"value":"2"},"wTotalLength":{"value":"0x001f"},"bNumInterfaces":{"value":"1"},"bConfigurationValue":{"value":"1"},"iConfiguration":{"value":"0"},"bmAttributes":{"value":"0xe0","attributes":["Self Powered","Remote Wakeup"]},"MaxPower":{"description":"0mA"},"interface_descriptors":[{"bLength":{"value":"9"},"bDescriptorType":{"value":"4"},"bInterfaceNumber":{"value":"0"},"bAlternateSetting":{"value":"0"},"bNumEndpoints":{"value":"1"},"bInterfaceClass":{"value":"9","description":"Hub"},"bInterfaceSubClass":{"value":"0"},"bInterfaceProtocol":{"value":"0","description":"Full speed (or root) hub"},"iInterface":{"value":"0"},"endpoint_descriptors":[{"bLength":{"value":"7"},"bDescriptorType":{"value":"5"},"bEndpointAddress":{"value":"0x81","description":"EP 1 IN"},"bmAttributes":{"value":"3","attributes":["Transfer Type Interrupt","Synch Type None","Usage Type Data"]},"wMaxPacketSize":{"value":"0x0004","description":"1x 4 bytes"},"bInterval":{"value":"12"},"bMaxBurst":{"value":"0"}}]}]}},"hub_descriptor":{"bLength":{"value":"12"},"bDescriptorType":{"value":"42"},"nNbrPorts":{"value":"4"},"wHubCharacteristic":{"value":"0x000a","attributes":["No power switching (usb 1.0)","Per-port overcurrent protection"]},"bPwrOn2PwrGood":{"value":"50 *","description":"2 milli seconds"},"bHubContrCurrent":{"value":"0","description":"milli Ampere"},"bHubDecLat":{"value":"0.0","description":"micro seconds"},"wHubDelay":{"value":"0","description":"nano seconds"},"DeviceRemovable":{"value":"0x00"},"hub_port_status":{"Port 1":{"value":"0000.02a0","attributes":["lowspeed","L1"]},"Port 2":{"value":"0000.02a0","attributes":["lowspeed","L1"]},"Port 3":{"value":"0000.02a0","attributes":["lowspeed","L1"]},"Port 4":{"value":"0000.02a0","attributes":["lowspeed","L1"]}}},"device_status":{"value":"0x0001","description":"Self Powered"}},{"bus":"003","device":"002","id":"2109:2812","description":"VIA Labs, Inc. VL812 Hub","device_descriptor":{"bLength":{"value":"18"},"bDescriptorType":{"value":"1"},"bcdUSB":{"value":"2.10"},"bDeviceClass":{"value":"9","description":"Hub"},"bDeviceSubClass":{"value":"0"},"bDeviceProtocol":{"value":"1","description":"Single TT"},"bMaxPacketSize0":{"value":"64"},"idVendor":{"value":"0x2109","description":"VIA Labs, Inc."},"idProduct":{"value":"0x2812","description":"VL812 Hub"},"bcdDevice":{"value":"b.e0"},"iManufacturer":{"value":"1","description":"VIA Labs, Inc."},"iProduct":{"value":"2","description":"USB2.0 Hub"},"iSerial":{"value":"0"},"bNumConfigurations":{"value":"1"},"configuration_descriptor":{"bLength":{"value":"9"},"bDescriptorType":{"value":"2"},"wTotalLength":{"value":"0x0019"},"bNumInterfaces":{"value":"1"},"bConfigurationValue":{"value":"1"},"iConfiguration":{"value":"0"},"bmAttributes":{"value":"0xe0","attributes":["Self Powered","Remote Wakeup"]},"MaxPower":{"description":"0mA"},"interface_descriptors":[{"bLength":{"value":"9"},"bDescriptorType":{"value":"4"},"bInterfaceNumber":{"value":"0"},"bAlternateSetting":{"value":"0"},"bNumEndpoints":{"value":"1"},"bInterfaceClass":{"value":"9","description":"Hub"},"bInterfaceSubClass":{"value":"0"},"bInterfaceProtocol":{"value":"0","description":"Full speed (or root) hub"},"iInterface":{"value":"0"},"endpoint_descriptors":[{"bLength":{"value":"7"},"bDescriptorType":{"value":"5"},"bEndpointAddress":{"value":"0x81","description":"EP 1 IN"},"bmAttributes":{"value":"3","attributes":["Transfer Type Interrupt","Synch Type None","Usage Type Data"]},"wMaxPacketSize":{"value":"0x0001","description":"1x 1 bytes"},"bInterval":{"value":"12"}}]}]}},"hub_descriptor":{"bLength":{"value":"9"},"bDescriptorType":{"value":"41"},"nNbrPorts":{"value":"4"},"wHubCharacteristic":{"value":"0x00e9","attributes":["Per-port power switching","Per-port overcurrent protection","TT think time 32 FS bits","Port indicators"]},"bPwrOn2PwrGood":{"value":"50 *","description":"2 milli seconds"},"bHubContrCurrent":{"value":"100","description":"milli Ampere"},"DeviceRemovable":{"value":"0x00"},"PortPwrCtrlMask":{"value":"0xff"},"hub_port_status":{"Port 1":{"value":"0000.0503","attributes":["highspeed","power","enable","connect"]},"Port 2":{"value":"0000.0100","attributes":["power"]},"Port 3":{"value":"0000.0103","attributes":["power","enable","connect"]},"Port 4":{"value":"0000.0100","attributes":["power"]}}},"device_status":{"value":"0x0001","description":"Self Powered"}},{"bus":"004","device":"001","id":"1d6b:0003","description":"Linux Foundation 3.0 root hub","device_descriptor":{"bLength":{"value":"18"},"bDescriptorType":{"value":"1"},"bcdUSB":{"value":"3.00"},"bDeviceClass":{"value":"9","description":"Hub"},"bDeviceSubClass":{"value":"0"},"bDeviceProtocol":{"value":"3"},"bMaxPacketSize0":{"value":"9"},"idVendor":{"value":"0x1d6b","description":"Linux Foundation"},"idProduct":{"value":"0x0003","description":"3.0 root hub"},"bcdDevice":{"value":"5.18"},"iManufacturer":{"value":"3","description":"Linux 5.18.13-200.fc36.x86_64 xhci-hcd"},"iProduct":{"value":"2","description":"xHCI Host Controller"},"iSerial":{"value":"1","description":"0000:00:14.0"},"bNumConfigurations":{"value":"1"},"configuration_descriptor":{"bLength":{"value":"9"},"bDescriptorType":{"value":"2"},"wTotalLength":{"value":"0x001f"},"bNumInterfaces":{"value":"1"},"bConfigurationValue":{"value":"1"},"iConfiguration":{"value":"0"},"bmAttributes":{"value":"0xe0","attributes":["Self Powered","Remote Wakeup"]},"MaxPower":{"description":"0mA"},"interface_descriptors":[{"bLength":{"value":"9"},"bDescriptorType":{"value":"4"},"bInterfaceNumber":{"value":"0"},"bAlternateSetting":{"value":"0"},"bNumEndpoints":{"value":"1"},"bInterfaceClass":{"value":"9","description":"Hub"},"bInterfaceSubClass":{"value":"0"},"bInterfaceProtocol":{"value":"0","description":"Full speed (or root) hub"},"iInterface":{"value":"0"},"endpoint_descriptors":[{"bLength":{"value":"7"},"bDescriptorType":{"value":"5"},"bEndpointAddress":{"value":"0x81","description":"EP 1 IN"},"bmAttributes":{"value":"3","attributes":["Transfer Type Interrupt","Synch Type None","Usage Type Data"]},"wMaxPacketSize":{"value":"0x0004","description":"1x 4 bytes"},"bInterval":{"value":"12"},"bMaxBurst":{"value":"0"}}]}]}},"hub_descriptor":{"bLength":{"value":"12"},"bDescriptorType":{"value":"42"},"nNbrPorts":{"value":"6"},"wHubCharacteristic":{"value":"0x000a","attributes":["No power switching (usb 1.0)","Per-port overcurrent protection"]},"bPwrOn2PwrGood":{"value":"50 *","description":"2 milli seconds"},"bHubContrCurrent":{"value":"0","description":"milli Ampere"},"bHubDecLat":{"value":"0.0","description":"micro seconds"},"wHubDelay":{"value":"0","description":"nano seconds"},"DeviceRemovable":{"value":"0x00"},"hub_port_status":{"Port 1":{"value":"0000.02a0","attributes":["5Gbps","power","Rx.Detect"]},"Port 2":{"value":"0000.02a0","attributes":["5Gbps","power","Rx.Detect"]},"Port 3":{"value":"0000.02a0","attributes":["5Gbps","power","Rx.Detect"]},"Port 4":{"value":"0000.02a0","attributes":["5Gbps","power","Rx.Detect"]},"Port 5":{"value":"0000.0203","attributes":["5Gbps","power","U0","enable","connect"]},"Port 6":{"value":"0000.02a0","attributes":["5Gbps","power","Rx.Detect"]}}},"device_status":{"value":"0x0001","description":"Self Powered"}},{"bus":"004","device":"002","id":"2109:0812","description":"VIA Labs, Inc. VL812 Hub","device_descriptor":{"bLength":{"value":"18"},"bDescriptorType":{"value":"1"},"bcdUSB":{"value":"3.00"},"bDeviceClass":{"value":"9","description":"Hub"},"bDeviceSubClass":{"value":"0"},"bDeviceProtocol":{"value":"3"},"bMaxPacketSize0":{"value":"9"},"idVendor":{"value":"0x2109","description":"VIA Labs, Inc."},"idProduct":{"value":"0x0812","description":"VL812 Hub"},"bcdDevice":{"value":"b.e1"},"iManufacturer":{"value":"1","description":"VIA Labs, Inc."},"iProduct":{"value":"2","description":"USB3.0 Hub"},"iSerial":{"value":"0"},"bNumConfigurations":{"value":"1"},"configuration_descriptor":{"bLength":{"value":"9"},"bDescriptorType":{"value":"2"},"wTotalLength":{"value":"0x001f"},"bNumInterfaces":{"value":"1"},"bConfigurationValue":{"value":"1"},"iConfiguration":{"value":"0"},"bmAttributes":{"value":"0xe0","attributes":["Self Powered","Remote Wakeup"]},"MaxPower":{"description":"0mA"},"interface_descriptors":[{"bLength":{"value":"9"},"bDescriptorType":{"value":"4"},"bInterfaceNumber":{"value":"0"},"bAlternateSetting":{"value":"0"},"bNumEndpoints":{"value":"1"},"bInterfaceClass":{"value":"9","description":"Hub"},"bInterfaceSubClass":{"value":"0"},"bInterfaceProtocol":{"value":"0","description":"Full speed (or root) hub"},"iInterface":{"value":"0"},"endpoint_descriptors":[{"bLength":{"value":"7"},"bDescriptorType":{"value":"5"},"bEndpointAddress":{"value":"0x81","description":"EP 1 IN"},"bmAttributes":{"value":"19","attributes":["Transfer Type Interrupt","Synch Type None","Usage Type Feedback"]},"wMaxPacketSize":{"value":"0x0002","description":"1x 2 bytes"},"bInterval":{"value":"8"},"bMaxBurst":{"value":"0"}}]}]}},"hub_descriptor":{"bLength":{"value":"12"},"bDescriptorType":{"value":"42"},"nNbrPorts":{"value":"4"},"wHubCharacteristic":{"value":"0x000d","attributes":["Per-port power switching","Compound device","Per-port overcurrent protection"]},"bPwrOn2PwrGood":{"value":"100 *","description":"2 milli seconds"},"bHubContrCurrent":{"value":"0","description":"milli Ampere"},"bHubDecLat":{"value":"0.4","description":"micro seconds"},"wHubDelay":{"value":"4004","description":"nano seconds"},"DeviceRemovable":{"value":"0x00"},"hub_port_status":{"Port 1":{"value":"0000.0263","attributes":["5Gbps","power","suspend","enable","connect"]},"Port 2":{"value":"0000.0203","attributes":["5Gbps","power","U0","enable","connect"]},"Port 3":{"value":"0000.02a0","attributes":["5Gbps","power","Rx.Detect"]},"Port 4":{"value":"0000.02a0","attributes":["5Gbps","power","Rx.Detect"]}}},"device_status":{"value":"0x000d","description":"U2 Enabled"}},{"bus":"004","device":"003","id":"2109:0812","description":"VIA Labs, Inc. VL812 Hub","device_descriptor":{"bLength":{"value":"18"},"bDescriptorType":{"value":"1"},"bcdUSB":{"value":"3.00"},"bDeviceClass":{"value":"9","description":"Hub"},"bDeviceSubClass":{"value":"0"},"bDeviceProtocol":{"value":"3"},"bMaxPacketSize0":{"value":"9"},"idVendor":{"value":"0x2109","description":"VIA Labs, Inc."},"idProduct":{"value":"0x0812","description":"VL812 Hub"},"bcdDevice":{"value":"b.e1"},"iManufacturer":{"value":"1","description":"VIA Labs, Inc."},"iProduct":{"value":"2","description":"USB3.0 Hub"},"iSerial":{"value":"0"},"bNumConfigurations":{"value":"1"},"configuration_descriptor":{"bLength":{"value":"9"},"bDescriptorType":{"value":"2"},"wTotalLength":{"value":"0x001f"},"bNumInterfaces":{"value":"1"},"bConfigurationValue":{"value":"1"},"iConfiguration":{"value":"0"},"bmAttributes":{"value":"0xe0","attributes":["Self Powered","Remote Wakeup"]},"MaxPower":{"description":"0mA"},"interface_descriptors":[{"bLength":{"value":"9"},"bDescriptorType":{"value":"4"},"bInterfaceNumber":{"value":"0"},"bAlternateSetting":{"value":"0"},"bNumEndpoints":{"value":"1"},"bInterfaceClass":{"value":"9","description":"Hub"},"bInterfaceSubClass":{"value":"0"},"bInterfaceProtocol":{"value":"0","description":"Full speed (or root) hub"},"iInterface":{"value":"0"},"endpoint_descriptors":[{"bLength":{"value":"7"},"bDescriptorType":{"value":"5"},"bEndpointAddress":{"value":"0x81","description":"EP 1 IN"},"bmAttributes":{"value":"19","attributes":["Transfer Type Interrupt","Synch Type None","Usage Type Feedback"]},"wMaxPacketSize":{"value":"0x0002","description":"1x 2 bytes"},"bInterval":{"value":"8"},"bMaxBurst":{"value":"0"}}]}]}},"hub_descriptor":{"bLength":{"value":"12"},"bDescriptorType":{"value":"42"},"nNbrPorts":{"value":"4"},"wHubCharacteristic":{"value":"0x000d","attributes":["Per-port power switching","Compound device","Per-port overcurrent protection"]},"bPwrOn2PwrGood":{"value":"100 *","description":"2 milli seconds"},"bHubContrCurrent":{"value":"0","description":"milli Ampere"},"bHubDecLat":{"value":"0.4","description":"micro seconds"},"wHubDelay":{"value":"4004","description":"nano seconds"},"DeviceRemovable":{"value":"0x00"},"hub_port_status":{"Port 1":{"value":"0000.02a0","attributes":["5Gbps","power","Rx.Detect"]},"Port 2":{"value":"0000.02a0","attributes":["5Gbps","power","Rx.Detect"]},"Port 3":{"value":"0000.02a0","attributes":["5Gbps","power","Rx.Detect"]},"Port 4":{"value":"0000.02a0","attributes":["5Gbps","power","Rx.Detect"]}}},"device_status":{"value":"0x000d","description":"U2 Enabled"}},{"bus":"004","device":"002","id":"0424:2744","description":"Microchip Technology, Inc. (formerly SMSC) Hub","device_descriptor":{"bLength":{"value":"18"},"bDescriptorType":{"value":"1"},"bcdUSB":{"value":"2.10"},"bDeviceClass":{"value":"9","description":"Hub"},"bDeviceSubClass":{"value":"0"},"bDeviceProtocol":{"value":"2","description":"TT per port"},"bMaxPacketSize0":{"value":"64"},"idVendor":{"value":"0x0424","description":"Microchip Technology, Inc. (formerly SMSC)"},"idProduct":{"value":"0x2744","description":"Hub"},"bcdDevice":{"value":"2.21"},"iManufacturer":{"value":"1","description":"Microchip Tech"},"iProduct":{"value":"2","description":"USB2744"},"iSerial":{"value":"0"},"bNumConfigurations":{"value":"1"},"configuration_descriptor":{"bLength":{"value":"9"},"bDescriptorType":{"value":"2"},"wTotalLength":{"value":"0x0029"},"bNumInterfaces":{"value":"1"},"bConfigurationValue":{"value":"1"},"iConfiguration":{"value":"0"},"bmAttributes":{"value":"0xe0","attributes":["Self Powered","Remote Wakeup"]},"MaxPower":{"description":"0mA"},"interface_descriptors":[{"bLength":{"value":"9"},"bDescriptorType":{"value":"4"},"bInterfaceNumber":{"value":"0"},"bAlternateSetting":{"value":"0"},"bNumEndpoints":{"value":"1"},"bInterfaceClass":{"value":"9","description":"Hub"},"bInterfaceSubClass":{"value":"0"},"bInterfaceProtocol":{"value":"1","description":"Single TT"},"iInterface":{"value":"0"},"endpoint_descriptors":[{"bLength":{"value":"7"},"bDescriptorType":{"value":"5"},"bEndpointAddress":{"value":"0x81","description":"EP 1 IN"},"bmAttributes":{"value":"3","attributes":["Transfer Type Interrupt","Synch Type None","Usage Type Data"]},"wMaxPacketSize":{"value":"0x0001","description":"1x 1 bytes"},"bInterval":{"value":"12"}}]},{"bLength":{"value":"9"},"bDescriptorType":{"value":"4"},"bInterfaceNumber":{"value":"0"},"bAlternateSetting":{"value":"1"},"bNumEndpoints":{"value":"1"},"bInterfaceClass":{"value":"9","description":"Hub"},"bInterfaceSubClass":{"value":"0"},"bInterfaceProtocol":{"value":"2","description":"TT per port"},"iInterface":{"value":"0"},"endpoint_descriptors":[{"bLength":{"value":"7"},"bDescriptorType":{"value":"5"},"bEndpointAddress":{"value":"0x81","description":"EP 1 IN"},"bmAttributes":{"value":"3","attributes":["Transfer Type Interrupt","Synch Type None","Usage Type Data"]},"wMaxPacketSize":{"value":"0x0001","description":"1x 1 bytes"},"bInterval":{"value":"12"}}]}]}},"hub_descriptor":{"bLength":{"value":"9"},"bDescriptorType":{"value":"41"},"nNbrPorts":{"value":"4"},"wHubCharacteristic":{"value":"0x000d","attributes":["Per-port power switching","Compound device","Per-port overcurrent protection","TT think time 8 FS bits"]},"bPwrOn2PwrGood":{"value":"50 *","description":"2 milli seconds"},"bHubContrCurrent":{"value":"0","description":"milli Ampere"},"DeviceRemovable":{"value":"0x10"},"PortPwrCtrlMask":{"value":"0xff"},"hub_port_status":{"Port 1":{"value":"0000.0507","attributes":["highspeed","power","suspend","enable","connect"]},"Port 2":{"value":"0000.0100","attributes":["power"]},"Port 3":{"value":"0000.0100","attributes":["power"]},"Port 4":{"value":"0000.0503","attributes":["highspeed","power","enable","connect"]}}},"device_status":{"value":"0x0001","description":"Self Powered"}},{"bus":"005","device":"001","id":"1d6b:0003","description":"Linux Foundation 3.0 root hub","device_descriptor":{"bLength":{"value":"18"},"bDescriptorType":{"value":"1"},"bcdUSB":{"value":"3.10"},"bDeviceClass":{"value":"9","description":"Hub"},"bDeviceSubClass":{"value":"0"},"bDeviceProtocol":{"value":"3"},"bMaxPacketSize0":{"value":"9"},"idVendor":{"value":"0x1d6b","description":"Linux Foundation"},"idProduct":{"value":"0x0003","description":"3.0 root hub"},"bcdDevice":{"value":"4.18"},"iManufacturer":{"value":"3","description":"Linux 4.18.0-372.16.1.el8_6.0.1.x86_64 xhci-hcd"},"iProduct":{"value":"2","description":"xHCI Host Controller"},"iSerial":{"value":"1","description":"0000:02:00.3"},"bNumConfigurations":{"value":"1"},"configuration_descriptor":{"bLength":{"value":"9"},"bDescriptorType":{"value":"2"},"wTotalLength":{"value":"0x001f"},"bNumInterfaces":{"value":"1"},"bConfigurationValue":{"value":"1"},"iConfiguration":{"value":"0"},"bmAttributes":{"value":"0xe0","attributes":["Self Powered","Remote Wakeup"]},"MaxPower":{"description":"0mA"},"interface_descriptors":[{"bLength":{"value":"9"},"bDescriptorType":{"value":"4"},"bInterfaceNumber":{"value":"0"},"bAlternateSetting":{"value":"0"},"bNumEndpoints":{"value":"1"},"bInterfaceClass":{"value":"9","description":"Hub"},"bInterfaceSubClass":{"value":"0"},"bInterfaceProtocol":{"value":"0","description":"Full speed (or root) hub"},"iInterface":{"value":"0"},"endpoint_descriptors":[{"bLength":{"value":"7"},"bDescriptorType":{"value":"5"},"bEndpointAddress":{"value":"0x81","description":"EP 1 IN"},"bmAttributes":{"value":"3","attributes":["Transfer Type Interrupt","Synch Type None","Usage Type Data"]},"wMaxPacketSize":{"value":"0x0004","description":"1x 4 bytes"},"bInterval":{"value":"12"},"bMaxBurst":{"value":"0"}}]}]}},"hub_descriptor":{"bLength":{"value":"12"},"bDescriptorType":{"value":"42"},"nNbrPorts":{"value":"2"},"wHubCharacteristic":{"value":"0x000a","attributes":["No power switching (usb 1.0)","Per-port overcurrent protection"]},"bPwrOn2PwrGood":{"value":"50 *","description":"2 milli seconds"},"bHubContrCurrent":{"value":"0","description":"milli Ampere"},"bHubDecLat":{"value":"0.0","description":"micro seconds"},"wHubDelay":{"value":"0","description":"nano seconds"},"DeviceRemovable":{"value":"0x00"},"hub_port_status":{"Port 1":{"value":"0000.02a0","attributes":["lowspeed","L1"]},"Port 2":{"value":"0000.0263","attributes":["lowspeed","L1","enable","connect"]}}},"device_status":{"value":"0x0001","description":"Self Powered"}},{"bus":"005","device":"002","id":"0424:5744","description":"Microchip Technology, Inc. (formerly SMSC) Hub","device_descriptor":{"bLength":{"value":"18"},"bDescriptorType":{"value":"1"},"bcdUSB":{"value":"3.20"},"bDeviceClass":{"value":"9","description":"Hub"},"bDeviceSubClass":{"value":"0"},"bDeviceProtocol":{"value":"3"},"bMaxPacketSize0":{"value":"9"},"idVendor":{"value":"0x0424","description":"Microchip Technology, Inc. (formerly SMSC)"},"idProduct":{"value":"0x5744","description":"Hub"},"bcdDevice":{"value":"2.21"},"iManufacturer":{"value":"2","description":"Microchip Tech"},"iProduct":{"value":"3","description":"USB5744"},"iSerial":{"value":"0"},"bNumConfigurations":{"value":"1"},"configuration_descriptor":{"bLength":{"value":"9"},"bDescriptorType":{"value":"2"},"wTotalLength":{"value":"0x001f"},"bNumInterfaces":{"value":"1"},"bConfigurationValue":{"value":"1"},"iConfiguration":{"value":"0"},"bmAttributes":{"value":"0xe0","attributes":["Self Powered","Remote Wakeup"]},"MaxPower":{"description":"0mA"},"interface_descriptors":[{"bLength":{"value":"9"},"bDescriptorType":{"value":"4"},"bInterfaceNumber":{"value":"0"},"bAlternateSetting":{"value":"0"},"bNumEndpoints":{"value":"1"},"bInterfaceClass":{"value":"9","description":"Hub"},"bInterfaceSubClass":{"value":"0"},"bInterfaceProtocol":{"value":"0","description":"Full speed (or root) hub"},"iInterface":{"value":"0"},"endpoint_descriptors":[{"bLength":{"value":"7"},"bDescriptorType":{"value":"5"},"bEndpointAddress":{"value":"0x81","description":"EP 1 IN"},"bmAttributes":{"value":"19","attributes":["Transfer Type Interrupt","Synch Type None","Usage Type Feedback"]},"wMaxPacketSize":{"value":"0x0002","description":"1x 2 bytes"},"bInterval":{"value":"8"},"bMaxBurst":{"value":"0"}}]}]}},"hub_descriptor":{"bLength":{"value":"12"},"bDescriptorType":{"value":"42"},"nNbrPorts":{"value":"3"},"wHubCharacteristic":{"value":"0x0009","attributes":["Per-port power switching","Per-port overcurrent protection"]},"bPwrOn2PwrGood":{"value":"48 *","description":"2 milli seconds"},"bHubContrCurrent":{"value":"0","description":"milli Ampere"},"bHubDecLat":{"value":"0.8","description":"micro seconds"},"wHubDelay":{"value":"2312","description":"nano seconds"},"DeviceRemovable":{"value":"0x00"},"hub_port_status":{"Port 1":{"value":"0000.06a0","attributes":["highspeed","lowspeed","L1"]},"Port 2":{"value":"0000.06a0","attributes":["highspeed","lowspeed","L1"]},"Port 3":{"value":"0000.06a0","attributes":["highspeed","lowspeed","L1"]}}},"device_status":{"value":"0x0001","description":"Self Powered"}}] diff --git a/tests/fixtures/generic/lsusb-binary-object-store.out b/tests/fixtures/generic/lsusb-binary-object-store.out new file mode 100644 index 000000000..47644895f --- /dev/null +++ b/tests/fixtures/generic/lsusb-binary-object-store.out @@ -0,0 +1,1373 @@ + +Bus 002 Device 001: ID 1d6b:0003 Linux Foundation 3.0 root hub +Device Descriptor: + bLength 18 + bDescriptorType 1 + bcdUSB 3.10 + bDeviceClass 9 Hub + bDeviceSubClass 0 + bDeviceProtocol 3 + bMaxPacketSize0 9 + idVendor 0x1d6b Linux Foundation + idProduct 0x0003 3.0 root hub + bcdDevice 4.18 + iManufacturer 3 Linux 4.18.0-372.9.1.el8.x86_64 xhci-hcd + iProduct 2 xHCI Host Controller + iSerial 1 0000:02:00.0 + bNumConfigurations 1 + Configuration Descriptor: + bLength 9 + bDescriptorType 2 + wTotalLength 0x001f + bNumInterfaces 1 + bConfigurationValue 1 + iConfiguration 0 + bmAttributes 0xe0 + Self Powered + Remote Wakeup + MaxPower 0mA + Interface Descriptor: + bLength 9 + bDescriptorType 4 + bInterfaceNumber 0 + bAlternateSetting 0 + bNumEndpoints 1 + bInterfaceClass 9 Hub + bInterfaceSubClass 0 + bInterfaceProtocol 0 Full speed (or root) hub + iInterface 0 + Endpoint Descriptor: + bLength 7 + bDescriptorType 5 + bEndpointAddress 0x81 EP 1 IN + bmAttributes 3 + Transfer Type Interrupt + Synch Type None + Usage Type Data + wMaxPacketSize 0x0004 1x 4 bytes + bInterval 12 + bMaxBurst 0 +Hub Descriptor: + bLength 12 + bDescriptorType 42 + nNbrPorts 4 + wHubCharacteristic 0x000a + No power switching (usb 1.0) + Per-port overcurrent protection + bPwrOn2PwrGood 50 * 2 milli seconds + bHubContrCurrent 0 milli Ampere + bHubDecLat 0.0 micro seconds + wHubDelay 0 nano seconds + DeviceRemovable 0x00 + Hub Port Status: + Port 1: 0000.02a0 lowspeed L1 + Port 2: 0000.02a0 lowspeed L1 + Port 3: 0000.02a0 lowspeed L1 + Port 4: 0000.02a0 lowspeed L1 +Binary Object Store Descriptor: + bLength 5 + bDescriptorType 15 + wTotalLength 0x002b + bNumDeviceCaps 2 + SuperSpeed USB Device Capability: + bLength 10 + bDescriptorType 16 + bDevCapabilityType 3 + bmAttributes 0x00 + wSpeedsSupported 0x0008 + Device can operate at SuperSpeed (5Gbps) + bFunctionalitySupport 1 + Lowest fully-functional device speed is Full Speed (12Mbps) + bU1DevExitLat 0 micro seconds + bU2DevExitLat 0 micro seconds + SuperSpeedPlus USB Device Capability: + bLength 28 + bDescriptorType 16 + bDevCapabilityType 10 + bmAttributes 0x00000023 + Sublink Speed Attribute count 3 + Sublink Speed ID count 1 + wFunctionalitySupport 0x1104 + bmSublinkSpeedAttr[0] 0x00050034 + Speed Attribute ID: 4 5Gb/s Symmetric RX SuperSpeed + bmSublinkSpeedAttr[1] 0x000500b4 + Speed Attribute ID: 4 5Gb/s Symmetric TX SuperSpeed + bmSublinkSpeedAttr[2] 0x000a4035 + Speed Attribute ID: 5 10Gb/s Symmetric RX SuperSpeedPlus + bmSublinkSpeedAttr[3] 0x000a40b5 + Speed Attribute ID: 5 10Gb/s Symmetric TX SuperSpeedPlus +Device Status: 0x0001 + Self Powered + +Bus 003 Device 002: ID 2109:2813 VIA Labs, Inc. VL813 Hub +Device Descriptor: + bLength 18 + bDescriptorType 1 + bcdUSB 2.10 + bDeviceClass 9 Hub + bDeviceSubClass 0 + bDeviceProtocol 1 Single TT + bMaxPacketSize0 64 + idVendor 0x2109 VIA Labs, Inc. + idProduct 0x2813 VL813 Hub + bcdDevice 90.11 + iManufacturer 1 VIA Labs, Inc. + iProduct 2 USB2.0 Hub + iSerial 0 + bNumConfigurations 1 + Configuration Descriptor: + bLength 9 + bDescriptorType 2 + wTotalLength 0x0019 + bNumInterfaces 1 + bConfigurationValue 1 + iConfiguration 0 + bmAttributes 0xe0 + Self Powered + Remote Wakeup + MaxPower 0mA + Interface Descriptor: + bLength 9 + bDescriptorType 4 + bInterfaceNumber 0 + bAlternateSetting 0 + bNumEndpoints 1 + bInterfaceClass 9 Hub + bInterfaceSubClass 0 + bInterfaceProtocol 0 Full speed (or root) hub + iInterface 0 + Endpoint Descriptor: + bLength 7 + bDescriptorType 5 + bEndpointAddress 0x81 EP 1 IN + bmAttributes 3 + Transfer Type Interrupt + Synch Type None + Usage Type Data + wMaxPacketSize 0x0001 1x 1 bytes + bInterval 12 +Hub Descriptor: + bLength 9 + bDescriptorType 41 + nNbrPorts 4 + wHubCharacteristic 0x00e9 + Per-port power switching + Per-port overcurrent protection + TT think time 32 FS bits + Port indicators + bPwrOn2PwrGood 50 * 2 milli seconds + bHubContrCurrent 100 milli Ampere + DeviceRemovable 0x00 + PortPwrCtrlMask 0xff + Hub Port Status: + Port 1: 0000.0503 highspeed power enable connect + Port 2: 0000.0100 power + Port 3: 0000.0100 power + Port 4: 0000.0100 power +Binary Object Store Descriptor: + bLength 5 + bDescriptorType 15 + wTotalLength 0x002a + bNumDeviceCaps 3 + USB 2.0 Extension Device Capability: + bLength 7 + bDescriptorType 16 + bDevCapabilityType 2 + bmAttributes 0x00000002 + HIRD Link Power Management (LPM) Supported + SuperSpeed USB Device Capability: + bLength 10 + bDescriptorType 16 + bDevCapabilityType 3 + bmAttributes 0x00 + wSpeedsSupported 0x000e + Device can operate at Full Speed (12Mbps) + Device can operate at High Speed (480Mbps) + Device can operate at SuperSpeed (5Gbps) + bFunctionalitySupport 1 + Lowest fully-functional device speed is Full Speed (12Mbps) + bU1DevExitLat 4 micro seconds + bU2DevExitLat 231 micro seconds + Container ID Device Capability: + bLength 20 + bDescriptorType 16 + bDevCapabilityType 4 + bReserved 0 + ContainerID {00000000-0000-0000-0000-000000000000} +Device Status: 0x0001 + Self Powered + +Bus 003 Device 003: ID 2109:2813 VIA Labs, Inc. VL813 Hub +Device Descriptor: + bLength 18 + bDescriptorType 1 + bcdUSB 2.10 + bDeviceClass 9 Hub + bDeviceSubClass 0 + bDeviceProtocol 1 Single TT + bMaxPacketSize0 64 + idVendor 0x2109 VIA Labs, Inc. + idProduct 0x2813 VL813 Hub + bcdDevice 90.11 + iManufacturer 1 VIA Labs, Inc. + iProduct 2 USB2.0 Hub + iSerial 0 + bNumConfigurations 1 + Configuration Descriptor: + bLength 9 + bDescriptorType 2 + wTotalLength 0x0019 + bNumInterfaces 1 + bConfigurationValue 1 + iConfiguration 0 + bmAttributes 0xe0 + Self Powered + Remote Wakeup + MaxPower 0mA + Interface Descriptor: + bLength 9 + bDescriptorType 4 + bInterfaceNumber 0 + bAlternateSetting 0 + bNumEndpoints 1 + bInterfaceClass 9 Hub + bInterfaceSubClass 0 + bInterfaceProtocol 0 Full speed (or root) hub + iInterface 0 + Endpoint Descriptor: + bLength 7 + bDescriptorType 5 + bEndpointAddress 0x81 EP 1 IN + bmAttributes 3 + Transfer Type Interrupt + Synch Type None + Usage Type Data + wMaxPacketSize 0x0001 1x 1 bytes + bInterval 12 +Hub Descriptor: + bLength 9 + bDescriptorType 41 + nNbrPorts 4 + wHubCharacteristic 0x00e9 + Per-port power switching + Per-port overcurrent protection + TT think time 32 FS bits + Port indicators + bPwrOn2PwrGood 50 * 2 milli seconds + bHubContrCurrent 100 milli Ampere + DeviceRemovable 0x00 + PortPwrCtrlMask 0xff + Hub Port Status: + Port 1: 0000.0503 highspeed power enable connect + Port 2: 0000.0100 power + Port 3: 0000.0100 power + Port 4: 0000.0100 power +Binary Object Store Descriptor: + bLength 5 + bDescriptorType 15 + wTotalLength 0x002a + bNumDeviceCaps 3 + USB 2.0 Extension Device Capability: + bLength 7 + bDescriptorType 16 + bDevCapabilityType 2 + bmAttributes 0x00000002 + HIRD Link Power Management (LPM) Supported + SuperSpeed USB Device Capability: + bLength 10 + bDescriptorType 16 + bDevCapabilityType 3 + bmAttributes 0x00 + wSpeedsSupported 0x000e + Device can operate at Full Speed (12Mbps) + Device can operate at High Speed (480Mbps) + Device can operate at SuperSpeed (5Gbps) + bFunctionalitySupport 1 + Lowest fully-functional device speed is Full Speed (12Mbps) + bU1DevExitLat 4 micro seconds + bU2DevExitLat 231 micro seconds + Container ID Device Capability: + bLength 20 + bDescriptorType 16 + bDevCapabilityType 4 + bReserved 0 + ContainerID {00000000-0000-0000-0000-000000000000} +Device Status: 0x0001 + Self Powered + +Bus 004 Device 001: ID 1d6b:0003 Linux Foundation 3.0 root hub +Device Descriptor: + bLength 18 + bDescriptorType 1 + bcdUSB 3.00 + bDeviceClass 9 Hub + bDeviceSubClass 0 + bDeviceProtocol 3 + bMaxPacketSize0 9 + idVendor 0x1d6b Linux Foundation + idProduct 0x0003 3.0 root hub + bcdDevice 4.18 + iManufacturer 3 Linux 4.18.0-372.9.1.el8.x86_64 xhci-hcd + iProduct 2 xHCI Host Controller + iSerial 1 0000:05:00.0 + bNumConfigurations 1 + Configuration Descriptor: + bLength 9 + bDescriptorType 2 + wTotalLength 0x001f + bNumInterfaces 1 + bConfigurationValue 1 + iConfiguration 0 + bmAttributes 0xe0 + Self Powered + Remote Wakeup + MaxPower 0mA + Interface Descriptor: + bLength 9 + bDescriptorType 4 + bInterfaceNumber 0 + bAlternateSetting 0 + bNumEndpoints 1 + bInterfaceClass 9 Hub + bInterfaceSubClass 0 + bInterfaceProtocol 0 Full speed (or root) hub + iInterface 0 + Endpoint Descriptor: + bLength 7 + bDescriptorType 5 + bEndpointAddress 0x81 EP 1 IN + bmAttributes 3 + Transfer Type Interrupt + Synch Type None + Usage Type Data + wMaxPacketSize 0x0004 1x 4 bytes + bInterval 12 + bMaxBurst 0 +Hub Descriptor: + bLength 12 + bDescriptorType 42 + nNbrPorts 2 + wHubCharacteristic 0x0009 + Per-port power switching + Per-port overcurrent protection + bPwrOn2PwrGood 50 * 2 milli seconds + bHubContrCurrent 0 milli Ampere + bHubDecLat 0.0 micro seconds + wHubDelay 0 nano seconds + DeviceRemovable 0x00 + Hub Port Status: + Port 1: 0000.0263 5Gbps power suspend enable connect + Port 2: 0000.02a0 5Gbps power Rx.Detect +Binary Object Store Descriptor: + bLength 5 + bDescriptorType 15 + wTotalLength 0x000f + bNumDeviceCaps 1 + SuperSpeed USB Device Capability: + bLength 10 + bDescriptorType 16 + bDevCapabilityType 3 + bmAttributes 0x02 + Latency Tolerance Messages (LTM) Supported + wSpeedsSupported 0x0008 + Device can operate at SuperSpeed (5Gbps) + bFunctionalitySupport 1 + Lowest fully-functional device speed is Full Speed (12Mbps) + bU1DevExitLat 0 micro seconds + bU2DevExitLat 0 micro seconds +Device Status: 0x0001 + Self Powered + +Bus 004 Device 002: ID 2109:0813 VIA Labs, Inc. VL813 Hub +Device Descriptor: + bLength 18 + bDescriptorType 1 + bcdUSB 3.00 + bDeviceClass 9 Hub + bDeviceSubClass 0 + bDeviceProtocol 3 + bMaxPacketSize0 9 + idVendor 0x2109 VIA Labs, Inc. + idProduct 0x0813 VL813 Hub + bcdDevice 90.11 + iManufacturer 1 VIA Labs, Inc. + iProduct 2 USB3.0 Hub + iSerial 0 + bNumConfigurations 1 + Configuration Descriptor: + bLength 9 + bDescriptorType 2 + wTotalLength 0x001f + bNumInterfaces 1 + bConfigurationValue 1 + iConfiguration 0 + bmAttributes 0xe0 + Self Powered + Remote Wakeup + MaxPower 0mA + Interface Descriptor: + bLength 9 + bDescriptorType 4 + bInterfaceNumber 0 + bAlternateSetting 0 + bNumEndpoints 1 + bInterfaceClass 9 Hub + bInterfaceSubClass 0 + bInterfaceProtocol 0 Full speed (or root) hub + iInterface 0 + Endpoint Descriptor: + bLength 7 + bDescriptorType 5 + bEndpointAddress 0x81 EP 1 IN + bmAttributes 19 + Transfer Type Interrupt + Synch Type None + Usage Type Feedback + wMaxPacketSize 0x0002 1x 2 bytes + bInterval 8 + bMaxBurst 0 +Hub Descriptor: + bLength 12 + bDescriptorType 42 + nNbrPorts 4 + wHubCharacteristic 0x0009 + Per-port power switching + Per-port overcurrent protection + bPwrOn2PwrGood 100 * 2 milli seconds + bHubContrCurrent 0 milli Ampere + bHubDecLat 0.4 micro seconds + wHubDelay 4004 nano seconds + DeviceRemovable 0x00 + Hub Port Status: + Port 1: 0000.0263 5Gbps power suspend enable connect + Port 2: 0000.02a0 5Gbps power Rx.Detect + Port 3: 0000.02a0 5Gbps power Rx.Detect + Port 4: 0000.02a0 5Gbps power Rx.Detect +Binary Object Store Descriptor: + bLength 5 + bDescriptorType 15 + wTotalLength 0x002a + bNumDeviceCaps 3 + USB 2.0 Extension Device Capability: + bLength 7 + bDescriptorType 16 + bDevCapabilityType 2 + bmAttributes 0x00000002 + HIRD Link Power Management (LPM) Supported + SuperSpeed USB Device Capability: + bLength 10 + bDescriptorType 16 + bDevCapabilityType 3 + bmAttributes 0x00 + wSpeedsSupported 0x000e + Device can operate at Full Speed (12Mbps) + Device can operate at High Speed (480Mbps) + Device can operate at SuperSpeed (5Gbps) + bFunctionalitySupport 1 + Lowest fully-functional device speed is Full Speed (12Mbps) + bU1DevExitLat 4 micro seconds + bU2DevExitLat 231 micro seconds + Container ID Device Capability: + bLength 20 + bDescriptorType 16 + bDevCapabilityType 4 + bReserved 0 + ContainerID {00000000-0000-0000-0000-000000000000} +Device Status: 0x0001 + Self Powered + +Bus 004 Device 003: ID 2109:0813 VIA Labs, Inc. VL813 Hub +Device Descriptor: + bLength 18 + bDescriptorType 1 + bcdUSB 3.00 + bDeviceClass 9 Hub + bDeviceSubClass 0 + bDeviceProtocol 3 + bMaxPacketSize0 9 + idVendor 0x2109 VIA Labs, Inc. + idProduct 0x0813 VL813 Hub + bcdDevice 90.11 + iManufacturer 1 VIA Labs, Inc. + iProduct 2 USB3.0 Hub + iSerial 0 + bNumConfigurations 1 + Configuration Descriptor: + bLength 9 + bDescriptorType 2 + wTotalLength 0x001f + bNumInterfaces 1 + bConfigurationValue 1 + iConfiguration 0 + bmAttributes 0xe0 + Self Powered + Remote Wakeup + MaxPower 0mA + Interface Descriptor: + bLength 9 + bDescriptorType 4 + bInterfaceNumber 0 + bAlternateSetting 0 + bNumEndpoints 1 + bInterfaceClass 9 Hub + bInterfaceSubClass 0 + bInterfaceProtocol 0 Full speed (or root) hub + iInterface 0 + Endpoint Descriptor: + bLength 7 + bDescriptorType 5 + bEndpointAddress 0x81 EP 1 IN + bmAttributes 19 + Transfer Type Interrupt + Synch Type None + Usage Type Feedback + wMaxPacketSize 0x0002 1x 2 bytes + bInterval 8 + bMaxBurst 0 +Hub Descriptor: + bLength 12 + bDescriptorType 42 + nNbrPorts 4 + wHubCharacteristic 0x0009 + Per-port power switching + Per-port overcurrent protection + bPwrOn2PwrGood 100 * 2 milli seconds + bHubContrCurrent 0 milli Ampere + bHubDecLat 0.4 micro seconds + wHubDelay 4004 nano seconds + DeviceRemovable 0x00 + Hub Port Status: + Port 1: 0000.02a0 5Gbps power Rx.Detect + Port 2: 0000.02a0 5Gbps power Rx.Detect + Port 3: 0000.02a0 5Gbps power Rx.Detect + Port 4: 0000.02a0 5Gbps power Rx.Detect +Binary Object Store Descriptor: + bLength 5 + bDescriptorType 15 + wTotalLength 0x002a + bNumDeviceCaps 3 + USB 2.0 Extension Device Capability: + bLength 7 + bDescriptorType 16 + bDevCapabilityType 2 + bmAttributes 0x00000002 + HIRD Link Power Management (LPM) Supported + SuperSpeed USB Device Capability: + bLength 10 + bDescriptorType 16 + bDevCapabilityType 3 + bmAttributes 0x00 + wSpeedsSupported 0x000e + Device can operate at Full Speed (12Mbps) + Device can operate at High Speed (480Mbps) + Device can operate at SuperSpeed (5Gbps) + bFunctionalitySupport 1 + Lowest fully-functional device speed is Full Speed (12Mbps) + bU1DevExitLat 4 micro seconds + bU2DevExitLat 231 micro seconds + Container ID Device Capability: + bLength 20 + bDescriptorType 16 + bDevCapabilityType 4 + bReserved 0 + ContainerID {00000000-0000-0000-0000-000000000000} +Device Status: 0x0001 + Self Powered + +Bus 006 Device 001: ID 1d6b:0003 Linux Foundation 3.0 root hub +Device Descriptor: + bLength 18 + bDescriptorType 1 + bcdUSB 3.10 + bDeviceClass 9 Hub + bDeviceSubClass 0 + bDeviceProtocol 3 + bMaxPacketSize0 9 + idVendor 0x1d6b Linux Foundation + idProduct 0x0003 3.0 root hub + bcdDevice 4.18 + iManufacturer 3 Linux 4.18.0-372.9.1.el8.x86_64 xhci-hcd + iProduct 2 xHCI Host Controller + iSerial 1 0000:0e:00.3 + bNumConfigurations 1 + Configuration Descriptor: + bLength 9 + bDescriptorType 2 + wTotalLength 0x001f + bNumInterfaces 1 + bConfigurationValue 1 + iConfiguration 0 + bmAttributes 0xe0 + Self Powered + Remote Wakeup + MaxPower 0mA + Interface Descriptor: + bLength 9 + bDescriptorType 4 + bInterfaceNumber 0 + bAlternateSetting 0 + bNumEndpoints 1 + bInterfaceClass 9 Hub + bInterfaceSubClass 0 + bInterfaceProtocol 0 Full speed (or root) hub + iInterface 0 + Endpoint Descriptor: + bLength 7 + bDescriptorType 5 + bEndpointAddress 0x81 EP 1 IN + bmAttributes 3 + Transfer Type Interrupt + Synch Type None + Usage Type Data + wMaxPacketSize 0x0004 1x 4 bytes + bInterval 12 + bMaxBurst 0 +Hub Descriptor: + bLength 12 + bDescriptorType 42 + nNbrPorts 4 + wHubCharacteristic 0x000a + No power switching (usb 1.0) + Per-port overcurrent protection + bPwrOn2PwrGood 50 * 2 milli seconds + bHubContrCurrent 0 milli Ampere + bHubDecLat 0.0 micro seconds + wHubDelay 0 nano seconds + DeviceRemovable 0x00 + Hub Port Status: + Port 1: 0000.02a0 lowspeed L1 + Port 2: 0000.02a0 lowspeed L1 + Port 3: 0000.02a0 lowspeed L1 + Port 4: 0000.02a0 lowspeed L1 +Binary Object Store Descriptor: + bLength 5 + bDescriptorType 15 + wTotalLength 0x0023 + bNumDeviceCaps 2 + SuperSpeed USB Device Capability: + bLength 10 + bDescriptorType 16 + bDevCapabilityType 3 + bmAttributes 0x02 + Latency Tolerance Messages (LTM) Supported + wSpeedsSupported 0x0008 + Device can operate at SuperSpeed (5Gbps) + bFunctionalitySupport 1 + Lowest fully-functional device speed is Full Speed (12Mbps) + bU1DevExitLat 0 micro seconds + bU2DevExitLat 0 micro seconds + SuperSpeedPlus USB Device Capability: + bLength 20 + bDescriptorType 16 + bDevCapabilityType 10 + bmAttributes 0x00000001 + Sublink Speed Attribute count 1 + Sublink Speed ID count 0 + wFunctionalitySupport 0x1104 + bmSublinkSpeedAttr[0] 0x00050034 + Speed Attribute ID: 4 5Gb/s Symmetric RX SuperSpeed + bmSublinkSpeedAttr[1] 0x000500b4 + Speed Attribute ID: 4 5Gb/s Symmetric TX SuperSpeed +Device Status: 0x0001 + Self Powered + +Bus 003 Device 002: ID 2109:2812 VIA Labs, Inc. VL812 Hub +Device Descriptor: + bLength 18 + bDescriptorType 1 + bcdUSB 2.10 + bDeviceClass 9 Hub + bDeviceSubClass 0 + bDeviceProtocol 1 Single TT + bMaxPacketSize0 64 + idVendor 0x2109 VIA Labs, Inc. + idProduct 0x2812 VL812 Hub + bcdDevice b.e0 + iManufacturer 1 VIA Labs, Inc. + iProduct 2 USB2.0 Hub + iSerial 0 + bNumConfigurations 1 + Configuration Descriptor: + bLength 9 + bDescriptorType 2 + wTotalLength 0x0019 + bNumInterfaces 1 + bConfigurationValue 1 + iConfiguration 0 + bmAttributes 0xe0 + Self Powered + Remote Wakeup + MaxPower 0mA + Interface Descriptor: + bLength 9 + bDescriptorType 4 + bInterfaceNumber 0 + bAlternateSetting 0 + bNumEndpoints 1 + bInterfaceClass 9 Hub + bInterfaceSubClass 0 + bInterfaceProtocol 0 Full speed (or root) hub + iInterface 0 + Endpoint Descriptor: + bLength 7 + bDescriptorType 5 + bEndpointAddress 0x81 EP 1 IN + bmAttributes 3 + Transfer Type Interrupt + Synch Type None + Usage Type Data + wMaxPacketSize 0x0001 1x 1 bytes + bInterval 12 +Hub Descriptor: + bLength 9 + bDescriptorType 41 + nNbrPorts 4 + wHubCharacteristic 0x00e9 + Per-port power switching + Per-port overcurrent protection + TT think time 32 FS bits + Port indicators + bPwrOn2PwrGood 50 * 2 milli seconds + bHubContrCurrent 100 milli Ampere + DeviceRemovable 0x00 + PortPwrCtrlMask 0xff + Hub Port Status: + Port 1: 0000.0503 highspeed power enable connect + Port 2: 0000.0100 power + Port 3: 0000.0103 power enable connect + Port 4: 0000.0100 power +Binary Object Store Descriptor: + bLength 5 + bDescriptorType 15 + wTotalLength 0x002a + bNumDeviceCaps 3 + USB 2.0 Extension Device Capability: + bLength 7 + bDescriptorType 16 + bDevCapabilityType 2 + bmAttributes 0x00000002 + HIRD Link Power Management (LPM) Supported + SuperSpeed USB Device Capability: + bLength 10 + bDescriptorType 16 + bDevCapabilityType 3 + bmAttributes 0x00 + wSpeedsSupported 0x000e + Device can operate at Full Speed (12Mbps) + Device can operate at High Speed (480Mbps) + Device can operate at SuperSpeed (5Gbps) + bFunctionalitySupport 1 + Lowest fully-functional device speed is Full Speed (12Mbps) + bU1DevExitLat 4 micro seconds + bU2DevExitLat 231 micro seconds + Container ID Device Capability: + bLength 20 + bDescriptorType 16 + bDevCapabilityType 4 + bReserved 0 + ContainerID {6970202d-1e44-4cde-8f73-78ec5964eca7} +Device Status: 0x0001 + Self Powered + +Bus 004 Device 001: ID 1d6b:0003 Linux Foundation 3.0 root hub +Device Descriptor: + bLength 18 + bDescriptorType 1 + bcdUSB 3.00 + bDeviceClass 9 Hub + bDeviceSubClass 0 + bDeviceProtocol 3 + bMaxPacketSize0 9 + idVendor 0x1d6b Linux Foundation + idProduct 0x0003 3.0 root hub + bcdDevice 5.18 + iManufacturer 3 Linux 5.18.13-200.fc36.x86_64 xhci-hcd + iProduct 2 xHCI Host Controller + iSerial 1 0000:00:14.0 + bNumConfigurations 1 + Configuration Descriptor: + bLength 9 + bDescriptorType 2 + wTotalLength 0x001f + bNumInterfaces 1 + bConfigurationValue 1 + iConfiguration 0 + bmAttributes 0xe0 + Self Powered + Remote Wakeup + MaxPower 0mA + Interface Descriptor: + bLength 9 + bDescriptorType 4 + bInterfaceNumber 0 + bAlternateSetting 0 + bNumEndpoints 1 + bInterfaceClass 9 Hub + bInterfaceSubClass 0 + bInterfaceProtocol 0 Full speed (or root) hub + iInterface 0 + Endpoint Descriptor: + bLength 7 + bDescriptorType 5 + bEndpointAddress 0x81 EP 1 IN + bmAttributes 3 + Transfer Type Interrupt + Synch Type None + Usage Type Data + wMaxPacketSize 0x0004 1x 4 bytes + bInterval 12 + bMaxBurst 0 +Hub Descriptor: + bLength 12 + bDescriptorType 42 + nNbrPorts 6 + wHubCharacteristic 0x000a + No power switching (usb 1.0) + Per-port overcurrent protection + bPwrOn2PwrGood 50 * 2 milli seconds + bHubContrCurrent 0 milli Ampere + bHubDecLat 0.0 micro seconds + wHubDelay 0 nano seconds + DeviceRemovable 0x00 + Hub Port Status: + Port 1: 0000.02a0 5Gbps power Rx.Detect + Port 2: 0000.02a0 5Gbps power Rx.Detect + Port 3: 0000.02a0 5Gbps power Rx.Detect + Port 4: 0000.02a0 5Gbps power Rx.Detect + Port 5: 0000.0203 5Gbps power U0 enable connect + Port 6: 0000.02a0 5Gbps power Rx.Detect +Binary Object Store Descriptor: + bLength 5 + bDescriptorType 15 + wTotalLength 0x000f + bNumDeviceCaps 1 + SuperSpeed USB Device Capability: + bLength 10 + bDescriptorType 16 + bDevCapabilityType 3 + bmAttributes 0x02 + Latency Tolerance Messages (LTM) Supported + wSpeedsSupported 0x0008 + Device can operate at SuperSpeed (5Gbps) + bFunctionalitySupport 1 + Lowest fully-functional device speed is Full Speed (12Mbps) + bU1DevExitLat 10 micro seconds + bU2DevExitLat 512 micro seconds +Device Status: 0x0001 + Self Powered + +Bus 004 Device 002: ID 2109:0812 VIA Labs, Inc. VL812 Hub +Device Descriptor: + bLength 18 + bDescriptorType 1 + bcdUSB 3.00 + bDeviceClass 9 Hub + bDeviceSubClass 0 + bDeviceProtocol 3 + bMaxPacketSize0 9 + idVendor 0x2109 VIA Labs, Inc. + idProduct 0x0812 VL812 Hub + bcdDevice b.e1 + iManufacturer 1 VIA Labs, Inc. + iProduct 2 USB3.0 Hub + iSerial 0 + bNumConfigurations 1 + Configuration Descriptor: + bLength 9 + bDescriptorType 2 + wTotalLength 0x001f + bNumInterfaces 1 + bConfigurationValue 1 + iConfiguration 0 + bmAttributes 0xe0 + Self Powered + Remote Wakeup + MaxPower 0mA + Interface Descriptor: + bLength 9 + bDescriptorType 4 + bInterfaceNumber 0 + bAlternateSetting 0 + bNumEndpoints 1 + bInterfaceClass 9 Hub + bInterfaceSubClass 0 + bInterfaceProtocol 0 Full speed (or root) hub + iInterface 0 + Endpoint Descriptor: + bLength 7 + bDescriptorType 5 + bEndpointAddress 0x81 EP 1 IN + bmAttributes 19 + Transfer Type Interrupt + Synch Type None + Usage Type Feedback + wMaxPacketSize 0x0002 1x 2 bytes + bInterval 8 + bMaxBurst 0 +Hub Descriptor: + bLength 12 + bDescriptorType 42 + nNbrPorts 4 + wHubCharacteristic 0x000d + Per-port power switching + Compound device + Per-port overcurrent protection + bPwrOn2PwrGood 100 * 2 milli seconds + bHubContrCurrent 0 milli Ampere + bHubDecLat 0.4 micro seconds + wHubDelay 4004 nano seconds + DeviceRemovable 0x00 + Hub Port Status: + Port 1: 0000.0263 5Gbps power suspend enable connect + Port 2: 0000.0203 5Gbps power U0 enable connect + Port 3: 0000.02a0 5Gbps power Rx.Detect + Port 4: 0000.02a0 5Gbps power Rx.Detect +Binary Object Store Descriptor: + bLength 5 + bDescriptorType 15 + wTotalLength 0x002a + bNumDeviceCaps 3 + USB 2.0 Extension Device Capability: + bLength 7 + bDescriptorType 16 + bDevCapabilityType 2 + bmAttributes 0x00000002 + HIRD Link Power Management (LPM) Supported + SuperSpeed USB Device Capability: + bLength 10 + bDescriptorType 16 + bDevCapabilityType 3 + bmAttributes 0x00 + wSpeedsSupported 0x000e + Device can operate at Full Speed (12Mbps) + Device can operate at High Speed (480Mbps) + Device can operate at SuperSpeed (5Gbps) + bFunctionalitySupport 1 + Lowest fully-functional device speed is Full Speed (12Mbps) + bU1DevExitLat 4 micro seconds + bU2DevExitLat 231 micro seconds + Container ID Device Capability: + bLength 20 + bDescriptorType 16 + bDevCapabilityType 4 + bReserved 0 + ContainerID {6970202d-1e44-4cde-8f73-78ec5964eca7} +Device Status: 0x000d + Self Powered + U1 Enabled + U2 Enabled + +Bus 004 Device 003: ID 2109:0812 VIA Labs, Inc. VL812 Hub +Device Descriptor: + bLength 18 + bDescriptorType 1 + bcdUSB 3.00 + bDeviceClass 9 Hub + bDeviceSubClass 0 + bDeviceProtocol 3 + bMaxPacketSize0 9 + idVendor 0x2109 VIA Labs, Inc. + idProduct 0x0812 VL812 Hub + bcdDevice b.e1 + iManufacturer 1 VIA Labs, Inc. + iProduct 2 USB3.0 Hub + iSerial 0 + bNumConfigurations 1 + Configuration Descriptor: + bLength 9 + bDescriptorType 2 + wTotalLength 0x001f + bNumInterfaces 1 + bConfigurationValue 1 + iConfiguration 0 + bmAttributes 0xe0 + Self Powered + Remote Wakeup + MaxPower 0mA + Interface Descriptor: + bLength 9 + bDescriptorType 4 + bInterfaceNumber 0 + bAlternateSetting 0 + bNumEndpoints 1 + bInterfaceClass 9 Hub + bInterfaceSubClass 0 + bInterfaceProtocol 0 Full speed (or root) hub + iInterface 0 + Endpoint Descriptor: + bLength 7 + bDescriptorType 5 + bEndpointAddress 0x81 EP 1 IN + bmAttributes 19 + Transfer Type Interrupt + Synch Type None + Usage Type Feedback + wMaxPacketSize 0x0002 1x 2 bytes + bInterval 8 + bMaxBurst 0 +Hub Descriptor: + bLength 12 + bDescriptorType 42 + nNbrPorts 4 + wHubCharacteristic 0x000d + Per-port power switching + Compound device + Per-port overcurrent protection + bPwrOn2PwrGood 100 * 2 milli seconds + bHubContrCurrent 0 milli Ampere + bHubDecLat 0.4 micro seconds + wHubDelay 4004 nano seconds + DeviceRemovable 0x00 + Hub Port Status: + Port 1: 0000.02a0 5Gbps power Rx.Detect + Port 2: 0000.02a0 5Gbps power Rx.Detect + Port 3: 0000.02a0 5Gbps power Rx.Detect + Port 4: 0000.02a0 5Gbps power Rx.Detect +Binary Object Store Descriptor: + bLength 5 + bDescriptorType 15 + wTotalLength 0x002a + bNumDeviceCaps 3 + USB 2.0 Extension Device Capability: + bLength 7 + bDescriptorType 16 + bDevCapabilityType 2 + bmAttributes 0x00000002 + HIRD Link Power Management (LPM) Supported + SuperSpeed USB Device Capability: + bLength 10 + bDescriptorType 16 + bDevCapabilityType 3 + bmAttributes 0x00 + wSpeedsSupported 0x000e + Device can operate at Full Speed (12Mbps) + Device can operate at High Speed (480Mbps) + Device can operate at SuperSpeed (5Gbps) + bFunctionalitySupport 1 + Lowest fully-functional device speed is Full Speed (12Mbps) + bU1DevExitLat 4 micro seconds + bU2DevExitLat 231 micro seconds + Container ID Device Capability: + bLength 20 + bDescriptorType 16 + bDevCapabilityType 4 + bReserved 0 + ContainerID {6970202d-1e44-4cde-8f73-78ec5964eca7} +Device Status: 0x000d + Self Powered + U1 Enabled + U2 Enabled + +Bus 004 Device 002: ID 0424:2744 Microchip Technology, Inc. (formerly SMSC) Hub +Device Descriptor: + bLength 18 + bDescriptorType 1 + bcdUSB 2.10 + bDeviceClass 9 Hub + bDeviceSubClass 0 + bDeviceProtocol 2 TT per port + bMaxPacketSize0 64 + idVendor 0x0424 Microchip Technology, Inc. (formerly SMSC) + idProduct 0x2744 Hub + bcdDevice 2.21 + iManufacturer 1 Microchip Tech + iProduct 2 USB2744 + iSerial 0 + bNumConfigurations 1 + Configuration Descriptor: + bLength 9 + bDescriptorType 2 + wTotalLength 0x0029 + bNumInterfaces 1 + bConfigurationValue 1 + iConfiguration 0 + bmAttributes 0xe0 + Self Powered + Remote Wakeup + MaxPower 0mA + Interface Descriptor: + bLength 9 + bDescriptorType 4 + bInterfaceNumber 0 + bAlternateSetting 0 + bNumEndpoints 1 + bInterfaceClass 9 Hub + bInterfaceSubClass 0 + bInterfaceProtocol 1 Single TT + iInterface 0 + Endpoint Descriptor: + bLength 7 + bDescriptorType 5 + bEndpointAddress 0x81 EP 1 IN + bmAttributes 3 + Transfer Type Interrupt + Synch Type None + Usage Type Data + wMaxPacketSize 0x0001 1x 1 bytes + bInterval 12 + Interface Descriptor: + bLength 9 + bDescriptorType 4 + bInterfaceNumber 0 + bAlternateSetting 1 + bNumEndpoints 1 + bInterfaceClass 9 Hub + bInterfaceSubClass 0 + bInterfaceProtocol 2 TT per port + iInterface 0 + Endpoint Descriptor: + bLength 7 + bDescriptorType 5 + bEndpointAddress 0x81 EP 1 IN + bmAttributes 3 + Transfer Type Interrupt + Synch Type None + Usage Type Data + wMaxPacketSize 0x0001 1x 1 bytes + bInterval 12 +Hub Descriptor: + bLength 9 + bDescriptorType 41 + nNbrPorts 4 + wHubCharacteristic 0x000d + Per-port power switching + Compound device + Per-port overcurrent protection + TT think time 8 FS bits + bPwrOn2PwrGood 50 * 2 milli seconds + bHubContrCurrent 0 milli Ampere + DeviceRemovable 0x10 + PortPwrCtrlMask 0xff + Hub Port Status: + Port 1: 0000.0507 highspeed power suspend enable connect + Port 2: 0000.0100 power + Port 3: 0000.0100 power + Port 4: 0000.0503 highspeed power enable connect +Binary Object Store Descriptor: + bLength 5 + bDescriptorType 15 + wTotalLength 0x002a + bNumDeviceCaps 3 + USB 2.0 Extension Device Capability: + bLength 7 + bDescriptorType 16 + bDevCapabilityType 2 + bmAttributes 0x0000a21e + BESL Link Power Management (LPM) Supported + BESL value 512 us + Deep BESL value 40960 us + SuperSpeed USB Device Capability: + bLength 10 + bDescriptorType 16 + bDevCapabilityType 3 + bmAttributes 0x00 + wSpeedsSupported 0x000e + Device can operate at Full Speed (12Mbps) + Device can operate at High Speed (480Mbps) + Device can operate at SuperSpeed (5Gbps) + bFunctionalitySupport 1 + Lowest fully-functional device speed is Full Speed (12Mbps) + bU1DevExitLat 10 micro seconds + bU2DevExitLat 231 micro seconds + Container ID Device Capability: + bLength 20 + bDescriptorType 16 + bDevCapabilityType 4 + bReserved 0 + ContainerID {00000000-0000-0000-0000-002231550364} +Device Status: 0x0001 + Self Powered + +Bus 005 Device 001: ID 1d6b:0003 Linux Foundation 3.0 root hub +Device Descriptor: + bLength 18 + bDescriptorType 1 + bcdUSB 3.10 + bDeviceClass 9 Hub + bDeviceSubClass 0 + bDeviceProtocol 3 + bMaxPacketSize0 9 + idVendor 0x1d6b Linux Foundation + idProduct 0x0003 3.0 root hub + bcdDevice 4.18 + iManufacturer 3 Linux 4.18.0-372.16.1.el8_6.0.1.x86_64 xhci-hcd + iProduct 2 xHCI Host Controller + iSerial 1 0000:02:00.3 + bNumConfigurations 1 + Configuration Descriptor: + bLength 9 + bDescriptorType 2 + wTotalLength 0x001f + bNumInterfaces 1 + bConfigurationValue 1 + iConfiguration 0 + bmAttributes 0xe0 + Self Powered + Remote Wakeup + MaxPower 0mA + Interface Descriptor: + bLength 9 + bDescriptorType 4 + bInterfaceNumber 0 + bAlternateSetting 0 + bNumEndpoints 1 + bInterfaceClass 9 Hub + bInterfaceSubClass 0 + bInterfaceProtocol 0 Full speed (or root) hub + iInterface 0 + Endpoint Descriptor: + bLength 7 + bDescriptorType 5 + bEndpointAddress 0x81 EP 1 IN + bmAttributes 3 + Transfer Type Interrupt + Synch Type None + Usage Type Data + wMaxPacketSize 0x0004 1x 4 bytes + bInterval 12 + bMaxBurst 0 +Hub Descriptor: + bLength 12 + bDescriptorType 42 + nNbrPorts 2 + wHubCharacteristic 0x000a + No power switching (usb 1.0) + Per-port overcurrent protection + bPwrOn2PwrGood 50 * 2 milli seconds + bHubContrCurrent 0 milli Ampere + bHubDecLat 0.0 micro seconds + wHubDelay 0 nano seconds + DeviceRemovable 0x00 + Hub Port Status: + Port 1: 0000.02a0 lowspeed L1 + Port 2: 0000.0263 lowspeed L1 enable connect +Binary Object Store Descriptor: + bLength 5 + bDescriptorType 15 + wTotalLength 0x0023 + bNumDeviceCaps 2 + SuperSpeed USB Device Capability: + bLength 10 + bDescriptorType 16 + bDevCapabilityType 3 + bmAttributes 0x02 + Latency Tolerance Messages (LTM) Supported + wSpeedsSupported 0x0008 + Device can operate at SuperSpeed (5Gbps) + bFunctionalitySupport 1 + Lowest fully-functional device speed is Full Speed (12Mbps) + bU1DevExitLat 0 micro seconds + bU2DevExitLat 0 micro seconds + SuperSpeedPlus USB Device Capability: + bLength 20 + bDescriptorType 16 + bDevCapabilityType 10 + bmAttributes 0x00000001 + Sublink Speed Attribute count 1 + Sublink Speed ID count 0 + wFunctionalitySupport 0x1104 + bmSublinkSpeedAttr[0] 0x00050034 + Speed Attribute ID: 4 5Gb/s Symmetric RX SuperSpeed + bmSublinkSpeedAttr[1] 0x000500b4 + Speed Attribute ID: 4 5Gb/s Symmetric TX SuperSpeed +Device Status: 0x0001 + Self Powered + +Bus 005 Device 002: ID 0424:5744 Microchip Technology, Inc. (formerly SMSC) Hub +Device Descriptor: + bLength 18 + bDescriptorType 1 + bcdUSB 3.20 + bDeviceClass 9 Hub + bDeviceSubClass 0 + bDeviceProtocol 3 + bMaxPacketSize0 9 + idVendor 0x0424 Microchip Technology, Inc. (formerly SMSC) + idProduct 0x5744 Hub + bcdDevice 2.21 + iManufacturer 2 Microchip Tech + iProduct 3 USB5744 + iSerial 0 + bNumConfigurations 1 + Configuration Descriptor: + bLength 9 + bDescriptorType 2 + wTotalLength 0x001f + bNumInterfaces 1 + bConfigurationValue 1 + iConfiguration 0 + bmAttributes 0xe0 + Self Powered + Remote Wakeup + MaxPower 0mA + Interface Descriptor: + bLength 9 + bDescriptorType 4 + bInterfaceNumber 0 + bAlternateSetting 0 + bNumEndpoints 1 + bInterfaceClass 9 Hub + bInterfaceSubClass 0 + bInterfaceProtocol 0 Full speed (or root) hub + iInterface 0 + Endpoint Descriptor: + bLength 7 + bDescriptorType 5 + bEndpointAddress 0x81 EP 1 IN + bmAttributes 19 + Transfer Type Interrupt + Synch Type None + Usage Type Feedback + wMaxPacketSize 0x0002 1x 2 bytes + bInterval 8 + bMaxBurst 0 +Hub Descriptor: + bLength 12 + bDescriptorType 42 + nNbrPorts 3 + wHubCharacteristic 0x0009 + Per-port power switching + Per-port overcurrent protection + bPwrOn2PwrGood 48 * 2 milli seconds + bHubContrCurrent 0 milli Ampere + bHubDecLat 0.8 micro seconds + wHubDelay 2312 nano seconds + DeviceRemovable 0x00 + Hub Port Status: + Port 1: 0000.06a0 highspeed lowspeed L1 + Port 2: 0000.06a0 highspeed lowspeed L1 + Port 3: 0000.06a0 highspeed lowspeed L1 +Binary Object Store Descriptor: + bLength 5 + bDescriptorType 15 + wTotalLength 0x002a + bNumDeviceCaps 3 + USB 2.0 Extension Device Capability: + bLength 7 + bDescriptorType 16 + bDevCapabilityType 2 + bmAttributes 0x0000a21e + BESL Link Power Management (LPM) Supported + BESL value 512 us + Deep BESL value 40960 us + SuperSpeed USB Device Capability: + bLength 10 + bDescriptorType 16 + bDevCapabilityType 3 + bmAttributes 0x00 + wSpeedsSupported 0x000e + Device can operate at Full Speed (12Mbps) + Device can operate at High Speed (480Mbps) + Device can operate at SuperSpeed (5Gbps) + bFunctionalitySupport 1 + Lowest fully-functional device speed is Full Speed (12Mbps) + bU1DevExitLat 10 micro seconds + bU2DevExitLat 231 micro seconds + Container ID Device Capability: + bLength 20 + bDescriptorType 16 + bDevCapabilityType 4 + bReserved 0 + ContainerID {00000000-0000-0000-0000-002231550364} +Device Status: 0x0001 + Self Powered diff --git a/tests/fixtures/generic/lsusb-device-qualifier.json b/tests/fixtures/generic/lsusb-device-qualifier.json new file mode 100644 index 000000000..79475e4cb --- /dev/null +++ b/tests/fixtures/generic/lsusb-device-qualifier.json @@ -0,0 +1 @@ +[{"bus":"002","device":"002","id":"8087:8000","description":"Intel Corp. Integrated Rate Matching Hub","device_descriptor":{"bLength":{"value":"18"},"bDescriptorType":{"value":"1"},"bcdUSB":{"value":"2.00"},"bDeviceClass":{"value":"9","description":"Hub"},"bDeviceSubClass":{"value":"0"},"bDeviceProtocol":{"value":"1","description":"Single TT"},"bMaxPacketSize0":{"value":"64"},"idVendor":{"value":"0x8087","description":"Intel Corp."},"idProduct":{"value":"0x8000","description":"Integrated Rate Matching Hub"},"bcdDevice":{"value":"0.04"},"iManufacturer":{"value":"0"},"iProduct":{"value":"0"},"iSerial":{"value":"0"},"bNumConfigurations":{"value":"1"},"configuration_descriptor":{"bLength":{"value":"9"},"bDescriptorType":{"value":"2"},"wTotalLength":{"value":"0x0019"},"bNumInterfaces":{"value":"1"},"bConfigurationValue":{"value":"1"},"iConfiguration":{"value":"0"},"bmAttributes":{"value":"0xe0","attributes":["Self Powered","Remote Wakeup"]},"MaxPower":{"description":"0mA"},"interface_descriptors":[{"bLength":{"value":"9"},"bDescriptorType":{"value":"4"},"bInterfaceNumber":{"value":"0"},"bAlternateSetting":{"value":"0"},"bNumEndpoints":{"value":"1"},"bInterfaceClass":{"value":"9","description":"Hub"},"bInterfaceSubClass":{"value":"0"},"bInterfaceProtocol":{"value":"0","description":"Full speed (or root) hub"},"iInterface":{"value":"0"},"endpoint_descriptors":[{"bLength":{"value":"7"},"bDescriptorType":{"value":"5"},"bEndpointAddress":{"value":"0x81","description":"EP 1 IN"},"bmAttributes":{"value":"3","attributes":["Transfer Type Interrupt","Synch Type None","Usage Type Data"]},"wMaxPacketSize":{"value":"0x0002","description":"1x 2 bytes"},"bInterval":{"value":"12"}}]}]}},"hub_descriptor":{"bLength":{"value":"11"},"bDescriptorType":{"value":"41"},"nNbrPorts":{"value":"8"},"wHubCharacteristic":{"value":"0x0009","attributes":["Per-port power switching","Per-port overcurrent protection","TT think time 8 FS bits"]},"bPwrOn2PwrGood":{"value":"0 *","description":"2 milli seconds"},"bHubContrCurrent":{"value":"0","description":"milli Ampere"},"DeviceRemovable":{"value":"0x00","description":"0x00"},"PortPwrCtrlMask":{"value":"0xff","description":"0xff"},"hub_port_status":{"Port 1":{"value":"0000.0100","attributes":["power"]},"Port 2":{"value":"0000.0100","attributes":["power"]},"Port 3":{"value":"0000.0100","attributes":["power"]},"Port 4":{"value":"0000.0100","attributes":["power"]},"Port 5":{"value":"0000.0100","attributes":["power"]},"Port 6":{"value":"0000.0100","attributes":["power"]},"Port 7":{"value":"0000.0100","attributes":["power"]},"Port 8":{"value":"0000.0100","attributes":["power"]}}},"device_qualifier":{"bLength":{"value":"10"},"bDescriptorType":{"value":"6"},"bcdUSB":{"value":"2.00"},"bDeviceClass":{"value":"9","description":"Hub"},"bDeviceSubClass":{"value":"0"},"bDeviceProtocol":{"value":"0","description":"Full speed (or root) hub"},"bMaxPacketSize0":{"value":"64"},"bNumConfigurations":{"value":"1"}},"device_status":{"value":"0x0001","description":"Self Powered"}}] diff --git a/tests/test_lsusb.py b/tests/test_lsusb.py index de9812eb9..1999763b3 100644 --- a/tests/test_lsusb.py +++ b/tests/test_lsusb.py @@ -29,6 +29,12 @@ def setUp(self): with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/lsusb-t.out'), 'r', encoding='utf-8') as f: self.generic_lsusb_t = f.read() + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/lsusb-device-qualifier.out'), 'r', encoding='utf-8') as f: + self.generic_lsusb_device_qualifier = f.read() + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/lsusb-binary-object-store.out'), 'r', encoding='utf-8') as f: + self.generic_lsusb_binary_object_store = f.read() + # output with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/lsusb.json'), 'r', encoding='utf-8') as f: self.centos_7_7_lsusb_json = json.loads(f.read()) @@ -45,6 +51,12 @@ def setUp(self): with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/lsusb-test-attributes2.json'), 'r', encoding='utf-8') as f: self.generic_lsusb_test_attributes2_json = json.loads(f.read()) + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/lsusb-device-qualifier.json'), 'r', encoding='utf-8') as f: + self.generic_lsusb_devicez_qualifier_json = json.loads(f.read()) + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/lsusb-binary-object-store.json'), 'r', encoding='utf-8') as f: + self.generic_lsusb_binary_object_store_json = json.loads(f.read()) + def test_lsusb_nodata(self): """ Test 'lsusb' with no data @@ -87,6 +99,18 @@ def test_lsusb_test_attributes2_generic(self): """ self.assertEqual(jc.parsers.lsusb.parse(self.generic_lsusb_test_attributes2, quiet=True), self.generic_lsusb_test_attributes2_json) + def test_lsusb_device_qualifier(self): + """ + Test 'lsusb -v' with device qualifier section + """ + self.assertEqual(jc.parsers.lsusb.parse(self.generic_lsusb_device_qualifier, quiet=True), self.generic_lsusb_devicez_qualifier_json) + + def test_lsusb_binary_object_store(self): + """ + Test 'lsusb -v' with binary object store section + """ + self.assertEqual(jc.parsers.lsusb.parse(self.generic_lsusb_binary_object_store, quiet=True), self.generic_lsusb_binary_object_store_json) + if __name__ == '__main__': unittest.main() From ff7f830f65fa2d494c56a3a29185f93a419a2bf3 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sat, 30 Jul 2022 12:14:10 -0700 Subject: [PATCH 030/116] add __main__.py entrypoint --- CHANGELOG | 4 +++- jc/__main__.py | 3 +++ 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 jc/__main__.py diff --git a/CHANGELOG b/CHANGELOG index b679f8102..6bad6dfae 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -2,8 +2,10 @@ jc changelog xxxxxxxx v1.20.5 - Add IP Address string parser -- Fix `lsusb` command parser for output containing a `Device Qualifier` section +- Fix `lsusb` command parser for output containing a `Device Qualifier` and + `Binary Object Store Descriptor` sections - Change LANG=C to LC_ALL=C in locale instructions +- Add `__main__.py` to package allowing `python -m jc` usage 20220723 v1.20.4 - Fix URL string parser path list for URLs ending in a forward slash diff --git a/jc/__main__.py b/jc/__main__.py new file mode 100644 index 000000000..58b4937a6 --- /dev/null +++ b/jc/__main__.py @@ -0,0 +1,3 @@ +import jc.cli + +jc.cli.main() From cd5547dfd8c03973ca3cbbc9201453c98f77483e Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sat, 30 Jul 2022 12:23:43 -0700 Subject: [PATCH 031/116] doc update --- CHANGELOG | 2 +- man/jc.1 | 14 +++++++++----- templates/manpage_template | 12 ++++++++---- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 6bad6dfae..7192d0e75 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -4,7 +4,7 @@ xxxxxxxx v1.20.5 - Add IP Address string parser - Fix `lsusb` command parser for output containing a `Device Qualifier` and `Binary Object Store Descriptor` sections -- Change LANG=C to LC_ALL=C in locale instructions +- Change `LANG=C` to `LC_ALL=C` in locale instructions - Add `__main__.py` to package allowing `python -m jc` usage 20220723 v1.20.4 diff --git a/man/jc.1 b/man/jc.1 index 485123e51..366b1da37 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -1,4 +1,4 @@ -.TH jc 1 2022-07-29 1.20.5 "JSON Convert" +.TH jc 1 2022-07-30 1.20.5 "JSON Convert" .SH NAME \fBjc\fP \- JSON Convert JSONifies the output of many CLI tools and file-types .SH SYNOPSIS @@ -758,19 +758,23 @@ Note: The application data directory follows the XDG Base Directory Specificatio .SH CAVEATS \fBLocale\fP -For best results set the \fBLANG\fP locale environment variable to \fBC\fP or \fBen_US.UTF-8\fP. For example, either by setting directly on the command-line: +For best results set the locale environment variables to \fBC\fP or +\fBen_US.UTF-8\fP by modifying the \fBLC_ALL\fP variable: .RS -$ LANG=C date | jc \fB--date\fP +$ LC_ALL=C date | jc \fB--date\fP .RE -or by exporting to the environment before running commands: +You can also set the locale variables individually: .RS $ export LANG=C + +$ export LC_NUMERIC=C .RE -On some older systems UTF-8 output will be downgraded to ASCII with \fB\\u\fP escape sequences if the \fBC\fP locale does not support UTF-8 encoding. +On some older systems UTF-8 output will be downgraded to ASCII with \fB\\u\fP +escape sequences if the \fBC\fP locale does not support UTF-8 encoding. \fBTimezones\fP diff --git a/templates/manpage_template b/templates/manpage_template index f5ed2d216..e027125a5 100644 --- a/templates/manpage_template +++ b/templates/manpage_template @@ -203,19 +203,23 @@ Note: The application data directory follows the XDG Base Directory Specificatio .SH CAVEATS \fBLocale\fP -For best results set the \fBLANG\fP locale environment variable to \fBC\fP or \fBen_US.UTF-8\fP. For example, either by setting directly on the command-line: +For best results set the locale environment variables to \fBC\fP or +\fBen_US.UTF-8\fP by modifying the \fBLC_ALL\fP variable: .RS -$ LANG=C date | jc \fB--date\fP +$ LC_ALL=C date | jc \fB--date\fP .RE -or by exporting to the environment before running commands: +You can also set the locale variables individually: .RS $ export LANG=C + +$ export LC_NUMERIC=C .RE -On some older systems UTF-8 output will be downgraded to ASCII with \fB\\u\fP escape sequences if the \fBC\fP locale does not support UTF-8 encoding. +On some older systems UTF-8 output will be downgraded to ASCII with \fB\\u\fP +escape sequences if the \fBC\fP locale does not support UTF-8 encoding. \fBTimezones\fP From bf93d60bf12780ddd838fbbd6bd051b1fc43b9a6 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sun, 31 Jul 2022 10:52:00 -0700 Subject: [PATCH 032/116] doc update --- README.md | 6 +++--- man/jc.1 | 4 ++-- templates/manpage_template | 2 +- templates/readme_template | 6 +++--- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index c61b0c026..3d1a7aecf 100644 --- a/README.md +++ b/README.md @@ -13,9 +13,9 @@ for an example. # JC JSON Convert -`jc` JSONifies the output of many CLI tools and file-types for easier parsing in -scripts. See the [**Parsers**](#parsers) section for supported commands and -file-types. +`jc` JSONifies the output of many CLI tools, file-types, and common strings +for easier parsing in scripts. See the [**Parsers**](#parsers) section for +supported commands, file-types, and strings. ```bash dig example.com | jc --dig ``` diff --git a/man/jc.1 b/man/jc.1 index 366b1da37..8657bd47f 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -1,4 +1,4 @@ -.TH jc 1 2022-07-30 1.20.5 "JSON Convert" +.TH jc 1 2022-07-31 1.20.5 "JSON Convert" .SH NAME \fBjc\fP \- JSON Convert JSONifies the output of many CLI tools and file-types .SH SYNOPSIS @@ -9,7 +9,7 @@ or "Magic" syntax: \fBjc\fP [OPTIONS] COMMAND .SH DESCRIPTION -\fBjc\fP JSONifies the output of many CLI tools and file-types for easier parsing in scripts. \fBjc\fP accepts piped input from \fBSTDIN\fP and outputs a JSON representation of the previous command's output to \fBSTDOUT\fP. Alternatively, the "Magic" syntax can be used by prepending \fBjc\fP to the command to be converted. Options can be passed to \fBjc\fP immediately before the command is given. (Note: "Magic" syntax does not support shell builtins or command aliases) +\fBjc\fP JSONifies the output of many CLI tools, file-types, and common strings for easier parsing in scripts. \fBjc\fP accepts piped input from \fBSTDIN\fP and outputs a JSON representation of the previous command's output to \fBSTDOUT\fP. Alternatively, the "Magic" syntax can be used by prepending \fBjc\fP to the command to be converted. Options can be passed to \fBjc\fP immediately before the command is given. (Note: "Magic" syntax does not support shell builtins or command aliases) .SH OPTIONS .B diff --git a/templates/manpage_template b/templates/manpage_template index e027125a5..5bb644b8f 100644 --- a/templates/manpage_template +++ b/templates/manpage_template @@ -9,7 +9,7 @@ or "Magic" syntax: \fBjc\fP [OPTIONS] COMMAND .SH DESCRIPTION -\fBjc\fP JSONifies the output of many CLI tools and file-types for easier parsing in scripts. \fBjc\fP accepts piped input from \fBSTDIN\fP and outputs a JSON representation of the previous command's output to \fBSTDOUT\fP. Alternatively, the "Magic" syntax can be used by prepending \fBjc\fP to the command to be converted. Options can be passed to \fBjc\fP immediately before the command is given. (Note: "Magic" syntax does not support shell builtins or command aliases) +\fBjc\fP JSONifies the output of many CLI tools, file-types, and common strings for easier parsing in scripts. \fBjc\fP accepts piped input from \fBSTDIN\fP and outputs a JSON representation of the previous command's output to \fBSTDOUT\fP. Alternatively, the "Magic" syntax can be used by prepending \fBjc\fP to the command to be converted. Options can be passed to \fBjc\fP immediately before the command is given. (Note: "Magic" syntax does not support shell builtins or command aliases) .SH OPTIONS .B diff --git a/templates/readme_template b/templates/readme_template index 1db104f2c..61fa32697 100644 --- a/templates/readme_template +++ b/templates/readme_template @@ -13,9 +13,9 @@ for an example. # JC JSON Convert -`jc` JSONifies the output of many CLI tools and file-types for easier parsing in -scripts. See the [**Parsers**](#parsers) section for supported commands and -file-types. +`jc` JSONifies the output of many CLI tools, file-types, and common strings +for easier parsing in scripts. See the [**Parsers**](#parsers) section for +supported commands, file-types, and strings. ```bash dig example.com | jc --dig ``` From 108e1b730e43f7b2ffe87473e9359f2fb90e7bb4 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sun, 31 Jul 2022 16:14:04 -0700 Subject: [PATCH 033/116] add plist parser --- README.md | 1 + completions/jc_bash_completion.sh | 2 +- completions/jc_zsh_completion.sh | 3 +- docs/parsers/plist.md | 62 ++ jc/lib.py | 1 + jc/parsers/plist.py | 93 ++ man/jc.1 | 5 + .../generic/plist-garageband-info.plist | Bin 0 -> 15081 bytes .../fixtures/generic/plist-safari-info.plist | 799 ++++++++++++++++++ 9 files changed, 964 insertions(+), 2 deletions(-) create mode 100644 docs/parsers/plist.md create mode 100644 jc/parsers/plist.py create mode 100644 tests/fixtures/generic/plist-garageband-info.plist create mode 100644 tests/fixtures/generic/plist-safari-info.plist diff --git a/README.md b/README.md index 3d1a7aecf..21f23ba6f 100644 --- a/README.md +++ b/README.md @@ -218,6 +218,7 @@ option. | ` --ping-s` | `ping` and `ping6` command streaming parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/ping_s) | | ` --pip-list` | `pip list` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/pip_list) | | ` --pip-show` | `pip show` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/pip_show) | +| ` --plist` | PLIST file parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/plist) | | ` --postconf` | `postconf -M` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/postconf) | | ` --ps` | `ps` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/ps) | | ` --route` | `route` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/route) | diff --git a/completions/jc_bash_completion.sh b/completions/jc_bash_completion.sh index cac8256d0..0e31b0008 100644 --- a/completions/jc_bash_completion.sh +++ b/completions/jc_bash_completion.sh @@ -4,7 +4,7 @@ _jc() jc_about_options jc_about_mod_options jc_help_options jc_special_options jc_commands=(acpi airport arp blkid chage cksum crontab date df dig dmidecode dpkg du env file finger free git gpg hciconfig id ifconfig iostat iptables iw jobs last lastb ls lsblk lsmod lsof lsusb md5 md5sum mount mpstat netstat nmcli ntpq pidstat ping ping6 pip pip3 postconf printenv ps route rpm rsync sfdisk sha1sum sha224sum sha256sum sha384sum sha512sum shasum ss stat sum sysctl systemctl systeminfo timedatectl top tracepath tracepath6 traceroute traceroute6 ufw uname update-alternatives upower uptime vdir vmstat w wc who xrandr zipinfo) - jc_parsers=(--acpi --airport --airport-s --arp --asciitable --asciitable-m --blkid --chage --cksum --crontab --crontab-u --csv --csv-s --date --df --dig --dir --dmidecode --dpkg-l --du --email-address --env --file --finger --free --fstab --git-log --git-log-s --gpg --group --gshadow --hash --hashsum --hciconfig --history --hosts --id --ifconfig --ini --iostat --iostat-s --ip-address --iptables --iso-datetime --iw-scan --jar-manifest --jobs --jwt --kv --last --ls --ls-s --lsblk --lsmod --lsof --lsusb --m3u --mount --mpstat --mpstat-s --netstat --nmcli --ntpq --passwd --pidstat --pidstat-s --ping --ping-s --pip-list --pip-show --postconf --ps --route --rpm-qi --rsync --rsync-s --sfdisk --shadow --ss --stat --stat-s --sysctl --systemctl --systemctl-lj --systemctl-ls --systemctl-luf --systeminfo --time --timedatectl --timestamp --top --top-s --tracepath --traceroute --ufw --ufw-appinfo --uname --update-alt-gs --update-alt-q --upower --uptime --url --vmstat --vmstat-s --w --wc --who --x509-cert --xml --xrandr --yaml --zipinfo) + jc_parsers=(--acpi --airport --airport-s --arp --asciitable --asciitable-m --blkid --chage --cksum --crontab --crontab-u --csv --csv-s --date --df --dig --dir --dmidecode --dpkg-l --du --email-address --env --file --finger --free --fstab --git-log --git-log-s --gpg --group --gshadow --hash --hashsum --hciconfig --history --hosts --id --ifconfig --ini --iostat --iostat-s --ip-address --iptables --iso-datetime --iw-scan --jar-manifest --jobs --jwt --kv --last --ls --ls-s --lsblk --lsmod --lsof --lsusb --m3u --mount --mpstat --mpstat-s --netstat --nmcli --ntpq --passwd --pidstat --pidstat-s --ping --ping-s --pip-list --pip-show --plist --postconf --ps --route --rpm-qi --rsync --rsync-s --sfdisk --shadow --ss --stat --stat-s --sysctl --systemctl --systemctl-lj --systemctl-ls --systemctl-luf --systeminfo --time --timedatectl --timestamp --top --top-s --tracepath --traceroute --ufw --ufw-appinfo --uname --update-alt-gs --update-alt-q --upower --uptime --url --vmstat --vmstat-s --w --wc --who --x509-cert --xml --xrandr --yaml --zipinfo) jc_options=(--force-color -C --debug -d --monochrome -m --pretty -p --quiet -q --raw -r --unbuffer -u --yaml-out -y) jc_about_options=(--about -a) jc_about_mod_options=(--pretty -p --yaml-out -y --monochrome -m --force-color -C) diff --git a/completions/jc_zsh_completion.sh b/completions/jc_zsh_completion.sh index f0aaae726..38614a4dd 100644 --- a/completions/jc_zsh_completion.sh +++ b/completions/jc_zsh_completion.sh @@ -94,7 +94,7 @@ _jc() { 'xrandr:run "xrandr" command with magic syntax.' 'zipinfo:run "zipinfo" command with magic syntax.' ) - jc_parsers=(--acpi --airport --airport-s --arp --asciitable --asciitable-m --blkid --chage --cksum --crontab --crontab-u --csv --csv-s --date --df --dig --dir --dmidecode --dpkg-l --du --email-address --env --file --finger --free --fstab --git-log --git-log-s --gpg --group --gshadow --hash --hashsum --hciconfig --history --hosts --id --ifconfig --ini --iostat --iostat-s --ip-address --iptables --iso-datetime --iw-scan --jar-manifest --jobs --jwt --kv --last --ls --ls-s --lsblk --lsmod --lsof --lsusb --m3u --mount --mpstat --mpstat-s --netstat --nmcli --ntpq --passwd --pidstat --pidstat-s --ping --ping-s --pip-list --pip-show --postconf --ps --route --rpm-qi --rsync --rsync-s --sfdisk --shadow --ss --stat --stat-s --sysctl --systemctl --systemctl-lj --systemctl-ls --systemctl-luf --systeminfo --time --timedatectl --timestamp --top --top-s --tracepath --traceroute --ufw --ufw-appinfo --uname --update-alt-gs --update-alt-q --upower --uptime --url --vmstat --vmstat-s --w --wc --who --x509-cert --xml --xrandr --yaml --zipinfo) + jc_parsers=(--acpi --airport --airport-s --arp --asciitable --asciitable-m --blkid --chage --cksum --crontab --crontab-u --csv --csv-s --date --df --dig --dir --dmidecode --dpkg-l --du --email-address --env --file --finger --free --fstab --git-log --git-log-s --gpg --group --gshadow --hash --hashsum --hciconfig --history --hosts --id --ifconfig --ini --iostat --iostat-s --ip-address --iptables --iso-datetime --iw-scan --jar-manifest --jobs --jwt --kv --last --ls --ls-s --lsblk --lsmod --lsof --lsusb --m3u --mount --mpstat --mpstat-s --netstat --nmcli --ntpq --passwd --pidstat --pidstat-s --ping --ping-s --pip-list --pip-show --plist --postconf --ps --route --rpm-qi --rsync --rsync-s --sfdisk --shadow --ss --stat --stat-s --sysctl --systemctl --systemctl-lj --systemctl-ls --systemctl-luf --systeminfo --time --timedatectl --timestamp --top --top-s --tracepath --traceroute --ufw --ufw-appinfo --uname --update-alt-gs --update-alt-q --upower --uptime --url --vmstat --vmstat-s --w --wc --who --x509-cert --xml --xrandr --yaml --zipinfo) jc_parsers_describe=( '--acpi:`acpi` command parser' '--airport:`airport -I` command parser' @@ -166,6 +166,7 @@ _jc() { '--ping-s:`ping` and `ping6` command streaming parser' '--pip-list:`pip list` command parser' '--pip-show:`pip show` command parser' + '--plist:PLIST file parser' '--postconf:`postconf -M` command parser' '--ps:`ps` command parser' '--route:`route` command parser' diff --git a/docs/parsers/plist.md b/docs/parsers/plist.md new file mode 100644 index 000000000..0e1869d57 --- /dev/null +++ b/docs/parsers/plist.md @@ -0,0 +1,62 @@ +[Home](https://kellyjonbrazil.github.io/jc/) + + +# jc.parsers.plist + +jc - JSON Convert PLIST file parser + +Converts binary and XML PLIST files. + +Usage (cli): + + $ cat file.plist | jc --plist + +Usage (module): + + import jc + result = jc.parse('plist', plist_command_output) + +Schema: + + [ + { + "plist": string, + "bar": boolean, + "baz": integer + } + ] + +Examples: + + $ plist | jc --plist -p + [] + + $ plist | jc --plist -p -r + [] + + + +### parse + +```python +def parse(data: Union[str, bytes], + raw: bool = False, + quiet: bool = False) -> Dict +``` + +Main text parsing function + +Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + +Returns: + + List of Dictionaries. Raw or processed structured data. + +### Parser Information +Compatibility: linux, darwin, cygwin, win32, aix, freebsd + +Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/jc/lib.py b/jc/lib.py index e554fbd95..4feff98f8 100644 --- a/jc/lib.py +++ b/jc/lib.py @@ -79,6 +79,7 @@ 'ping-s', 'pip-list', 'pip-show', + 'plist', 'postconf', 'ps', 'route', diff --git a/jc/parsers/plist.py b/jc/parsers/plist.py new file mode 100644 index 000000000..3578ac590 --- /dev/null +++ b/jc/parsers/plist.py @@ -0,0 +1,93 @@ +"""jc - JSON Convert PLIST file parser + +Converts binary and XML PLIST files. + +Usage (cli): + + $ cat file.plist | jc --plist + +Usage (module): + + import jc + result = jc.parse('plist', plist_command_output) + +Schema: + + [ + { + "plist": string, + "bar": boolean, + "baz": integer + } + ] + +Examples: + + $ plist | jc --plist -p + [] + + $ plist | jc --plist -p -r + [] +""" +from typing import List, Dict, Union +import plistlib +import jc.utils + + +class info(): + """Provides parser metadata (version, author, etc.)""" + version = '1.0' + description = 'PLIST file parser' + author = 'Kelly Brazil' + author_email = 'kellyjonbrazil@gmail.com' + compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd'] + + +__version__ = info.version + + +def _process(proc_data: Dict) -> Dict: + """ + Final processing to conform to the schema. + + Parameters: + + proc_data: (List of Dictionaries) raw structured data to process + + Returns: + + List of Dictionaries. Structured to conform to the schema. + """ + return proc_data + + +def parse( + data: Union[str, bytes], + raw: bool = False, + quiet: bool = False +) -> Dict: + """ + Main text parsing function + + Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + + Returns: + + List of Dictionaries. Raw or processed structured data. + """ + jc.utils.compatibility(__name__, info.compatible, quiet) + + if isinstance(data, str): + data = bytes(data, 'utf-8') + + raw_output: Dict = {} + + if jc.utils.has_data(data): + + raw_output = plistlib.loads(data) + + return raw_output if raw else _process(raw_output) diff --git a/man/jc.1 b/man/jc.1 index 8657bd47f..faaf14de8 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -367,6 +367,11 @@ M3U and M3U8 file parser \fB--pip-show\fP `pip show` command parser +.TP +.B +\fB--plist\fP +PLIST file parser + .TP .B \fB--postconf\fP diff --git a/tests/fixtures/generic/plist-garageband-info.plist b/tests/fixtures/generic/plist-garageband-info.plist new file mode 100644 index 0000000000000000000000000000000000000000..9b50a348b0b6be4cefadc115c6f5eb9e9c6ee003 GIT binary patch literal 15081 zcmd5@2Yggj*1zYb_enD8?WOn0OcDZNK~2R-n!HJX5E3P}+uCMBell#z0xAQeQ3-<70_RFg@h20u?BwQxRM02jhVa524u z-bvTdyXfEPdU_ANm)=J=(EI5FbR&I`zC^dt?et5cq+io-=(qGc`aL~Nf1p3opXiYQ z5&!{IBa^qYbry?(XLVM=VsA2WS_7}w$TC{lHiyA3U{8z5X6H;g-qvX~4+vVpF7-Dw z*E>vlvA56MDCq2h*~D3gx_PS&KWSvKjp|MVXYVsxjovnycaF|%Y8DK9ucx7%GxBpA z)okNT@smcD*UqvIi^XiU+v*&4vyrn4x*1l^G9c(|>|(*L8xTzWy|eg5g7~&Z&W>|Z zyVNa4aSE?j4_Wv|UT5H}IMHmfX=Ji?w$mz@bb^I5G#EG=Hl?&mgbO-pOardbZ?+DJ zQ#G=9&&C=Bo5jElxpo!nX=MlSHLkVdnySRX@N1~Z%m$Ti}oJ=3Kn-AJ)^+-kpnrwa>S?|~GzA1|%m#)(ky9jx@!Tkq&^Fe3;i z+=w3)ot$nV*UyVv>upre(V6wQ1=rp@?{DN6@dmR+q+k}`FCAfw?+D$aV?3k1N1D-A zCvuM)#AE48s#E);+-x@4ZJnHbKqE_RXS=k*Sq{N&cGw!QokKdTNSGJV8N6L&E*BbS zcn(`_#evdkHw}(=+KC-(or|9lN5oz*2_hdz%#J52vUsP+s5U!~e0@ec+lFhl3tEvU zHL`Sk7Iap#WdL#PvLSGdyiJl0X^l7!2rYU9OXw4L>ufiu_ySMosl35bZ#FMaeNJ z^IZ)sZk6F3;X@yljcd%PV@hPY=Oi$jY%tI%_HZod07u8-V+ zQX_Ag!&X*{O3x|@v;+qdt1}xFoTw=k26I17<_B%_aVR5rNA4su$f=}-wBjaxqs=zE zq*GGdFX}_&O{nF{`9XZ4sK18EX=J8aTu`lG>t_y=Hqy@G6Km5h93h?LbTW&uL`}NL z95R>mka=DZP9RL(WcGwGCAEqubQ&D}f{Eqrc9b+XQW-wlT}WA^K{nSQjZ7iUWa=Dx z!OVyYl((1|_Y#Z+<<2VbcDzd%kpnpUPSQ)xAoGdlI8jwMwbin$#b)14IHDywq9;5V zAObma4EVfFznZQ3NWVW`sG;2F&53m4MdjcO7}lFFigE@izTR64w$Nu@i?sk;lkRtdA^< z>&`)a?rU?gL%(i7uSGeVfx4fwG7Wf{>X}wSYsE9)L59fL6H(hG%^o4=lJm$C@>g;` zxqvJs%g80(&xPv3)Z)7F8f?P?iB3F}D879||YbV4-j>eh8 zVB^t7K>NhTyHP0fp&${hV3Bde3m304-g?v=jf|1H8Bags9>923we2D|kQ>QOrWdTz^9Wf>ZYOt;JIOk77rBSr zOYS2ZPCU$O$=&4dWc`FdA1i~M8d;_f2Z>Jyw%_mJ7l>=SA$Ow?@Py=oeihG!R8ImK9qV$yxFew2f?^n4p*Edl!>tj*uN>C)q`IlM%9q>?5y| z*U0N$T27dL9VUCpEB^EzbHT_YB{7p9YA_p(Xse>Xkm^Ik#o7z`A#4(RHL`qZJcAP$ z&x|omjQ!*d@+LWeB>GnKD2(y1UUln29Xnfm>q4Dvgd8MqlXu9wJ|_SD zJuo{&KG4Xrefu_;b(}#sTXY)AB|}pyIqGO8`^LC$=N|OMbOUZFN%M7kxj|tO2k;1U zjE}jbNju0Va#9s_HOby`GR~&z9L_f@5uM$@ED+Yn|`I743lrjw>@snK4>)f zGm%E-N10@el{?TOEVoGFj<85=jM#>};ZgxVkRKvW2mnwBfItX>PzZx? zi15yTxYfLXE7j z1DlyPv6*T1G^4YK-m`9iGog-lyEl*q>5u`;aUwDdnUJOKXlrJ~UoFFs4LPjY*r&!H zVK?MKJ`_M96hSGJLHP;5XBdj1tYv&?S$Dd2j)WWnD~x=dk;eo%)=-_RkPZLzgUCKJOcAU0}FryE$E;h20(zZ!UexE zKx;t{JoNcPbd2S5?B$KDCxI~9tvn{O;PxudZ~^8Xyb~6J0gQk#(c{v?>XY%S%OHk+ zM!*7Rffa0E2L}wo5S$I?{2t7-2o_I(l#9*=ukD}fyJn+yo9!0oX*n|*#mYrkLk+-GrMoUt6^l~P7RVRYC8 zonJiG@8brU43@&pumPTg-S9qqOT%dfRnj`zN%eG)UPf2%gJp0DTnd-La##VE!{1;f zTme@TC9Hz$;CfgMH^NQmJ9r}0V)V7i6PXnA4E`M*tk^ADgRRo177XS~sRfNJ88a-N zn7W-scS=e(py7?+_%GoKh!2Blz)CqTX z85bkbJ^(X(0Jd=`|800eW~-VvVuDP}za{zlJ%C6G+zIz+WW{3wibpL?Rv5(v#oUwG z>PEMxvCp&0Ubr6~fQ|4VJOrEIAMh}O`WXBZ9)~9Y0}bPH?y-=L;-nM8h)#IK<@AKP z^)1HmL=Vc((Q?ASMtLO1_*&iF0>4;H4js#1Pr=jh3~YvHVGBG5Tj6X`gID1-cpctEzz)LO@D9B9CxmMl_QM-2C-kZDr_Tuo)O9V*&4^Kq_}85? z5i>|AZ_33B4PWX(~aebj~Wj?2*oVtxccRf5-BGI)~2s?(f~E9uk1tSNMp?y;_(H=-OdLy zCZk|DyTt_$#i>#jLND|SdMV_tz=cQi3yq+WG>S&k7%HQ&G>#@9ILS1Hrqc93AvnV{ zp32$9+#)sp@J^bjo?X}76gqb{KDqHYid(SX4M&>$;x~%ff1A0_t`OG{-Snd%$)s77 zq1iNt=F&WxPYV%}Qd&mKX~mxq5+^MX)1bq&h!(3m*(SA=85P;aW%NoZSi!`&jbYm; zwmCMz>!b=S5AxmT1}PgpD#z;)EUKcFw2D^KNwkJerc)56ztCxPIz5S=G67J3tgfN8 zqG|5YH5gfSp)}el7=_hnm%~X<*2sQ|*WpHK53Q#Sw2?N^W;%nOiWtqLt+b7H{Bqm- zx4yKUv_;)oSKo>}B6m^WQqjQ}I4y5*Gl$FbZ0Tqf+uY*Q?$##$MG}Hpnq~8H#OZVv zWvQBW(QZ1M&Y|;=UJYG9IjWn0BmO8=J51-&9$(gPnD){$r2L=fTF&~Jd?W&RR1VV( z7AC-tEA^>v$6}|idM#Z=uRnp1{4y`zNw0DhbNF1$F-t<6^t$niNZdz1LT{iq(wpeb^cH$6y^T8Q zFui>O`hI(LxkgF|kC##QrJ|j5Eoy|L?@lq9EFD&tYI0-sWUO-VjgW#rV%?cd(N>35 z4pRN7ts6`6suXAf-rbEApdIX-*{YiQ%9=?yH_hjUGg2ROE`#-j43>u4)ODSmtzESZ z)k^K`CKG1wZ3FJ7>e?wvMQx2nmh*G3o>8xnl`9k#-kgT5!f#^=Zw92oyV+gzA-akF zgFZ~3pwHrc+FaE-d+uqIXS7XL_+I7@H5`^{zV3R%qhcvzt}F%` z!q^TI`fPH)!$G@}!Me&0b{G9OeUd(f+k1v?9`B96%l7hQVI3s=k_ump-Ymrk=sNPz!z=a6JZVI89hEsE}F>*v4Q5re_74^u%K| z94}Sd@kHO-R@Wxh+45MHYaOO9(}R2Hf9MXnlkTFs=?L9J_t97BYxH%x|2GVfkC{#$ zuk4lXrLSOEz$b9-3V@%Wj1%_@|zwDq7r1glX z#|xhxPwZ49%km4&GJeX&7pDEi2k}4p`9ufdq+hrq#G|wti#44*1`+tausLGE2nLBA z8rjWiGlS=d1u3qqD4Kj0hh0qkGFBd=GI}f?z&5`PmoYjl&|b)N%w&qhAy})1 z1vA&|V04^`;V@k{)}mrC8Z{SI6iY?&VhxnL+``73gvlJ8!z$Q^m;&q&7IC^EbY>HM zQsHAol~S#&t;D;?uXx12S22JU{ur0@wMi|q0KfE1msl}hDRHYtc1mrvzDirGQu32j z)jI8Dl}cY(GexVLq_3S+rL3;eRcW;~m9_k2B{ykOO*L1otkPC#_57qNeI++}k{0jw zBJCuuYI02#&-2<^onBd`S5{VQ_1Y=b+FJdjT5*uFN;O4WHR>Km>ZEFao5EmNWK`h4 z#b9bxrBUL4Dm{l~Pd{NHK_xW7Ea=7mD7q5!6i;H9^m7bfeC?SflYzEfoBCS54<(--oVEKpAFm+ z_*~$Nf!hNQ1b!3*K{O~is4l2GNC+AXS`oA+=#ij*1w9}1O3kS(WTM~9n z*qvdU!nTIJ9`;q(58Hg@F&Ax4nG+Fbwo^rJc5a+ zh-i**M68auBjS;W7bD(}_%Pzrh|eOviugX_NTfWnFtR4HCGw2OMUm%4UKY7J@}9_h zBezE$hOi|3Vn68+fn7$ZC%%w4R#B7V%6Z3&A zNER-OlNHLQ%35T-GK=go*-F_)*%sLj*{87(8yuSxn;%;mJ1w>&RvWu0_KMhRW7ov4 zk9|INf9%27pW+}cJT5D)GHym(Pn;?4lDNy`R>wUU_i=n${N(sk;@Nm}yfuDt{MqqK z;;)Dwj^7mjeEh-qBl0YHi+q-xmtP=XCSM`HQoc%lgZx(c2Km$S5%~uRG$AA*K0%RC zpKy9YPl7SwvV_YMu1dHz;rfKT68@R6J>g)&;l$9y^u)r%>ckm|J&B7FhZ2`0UX-{p z@utMv6YoyEH}QeQO^Gig?oa$U@kmm9QdyEJX?jvml8|(M(u$;Olh!3Yle9HyThh*? zJxL!W{gfP)%p_MQwZFAbbwAa%Pr_*$K`lR&s^abhXq_0fBBmJ@TZRu~OA51@# z{z3X@8M2I$jPi`R84EHNW?Y)#%y=+kOUC|;&ohHElQL5?nau3Wip;5*(=+E~YBTMb zt1|D)+?=^5^Mfpsm5^1GH8pE`R$W$OR!i1tS(>cHS!ZXh%vzPTE^Bkv{;V&UASRiq zU>cd(OfR#5;h6!($e5V(nH!l~mGdWvw-pVpL++!wJ99^Jzsx$ z6@FCsS>acO-xnS!3M>jLDl9swsIJITfWKO6yCTN@td~ma?U1mKsZ!mR?@!EPbqWTj@Kc-<3s{#gxg*%F1fW zYRfvyy2|F2Eh$@Cc4^raW!IN&DtoN#$+E3w+spQp9WE~|pHkjiesTFd<&T!Xr-)Zn zD3pq7#Z1K<#Ztva#dgKZid~9BimxhiDmp3_Rv0QQ73V6Wl&Q)r>N3@^YLjZSYOm^}N>Z6pnORv`*-v z@<8RgRZtaJ6<(EHl~Yw(^+MI#RfnpBtCOo!tMjW%tCiKZ>RXBQMXBp=K!EG7=Xdop F{|7!GJ8b{} literal 0 HcmV?d00001 diff --git a/tests/fixtures/generic/plist-safari-info.plist b/tests/fixtures/generic/plist-safari-info.plist new file mode 100644 index 000000000..2847c5694 --- /dev/null +++ b/tests/fixtures/generic/plist-safari-info.plist @@ -0,0 +1,799 @@ + + + + + ASWebAuthenticationSessionWebBrowserSupportCapabilities + + EphemeralBrowserSessionIsSupported + + IsSupported + + + Application-Group + + dot-mac + InternetAccounts + + BuildMachineOSBuild + 20A241133 + CFBundleDevelopmentRegion + English + CFBundleDisplayName + Safari + CFBundleDocumentTypes + + + CFBundleTypeExtensions + + css + + CFBundleTypeIconFile + css.icns + CFBundleTypeIconSystemGenerated + YES + CFBundleTypeMIMETypes + + text/css + + CFBundleTypeName + CSS style sheet + CFBundleTypeRole + Viewer + LSHandlerRank + Alternate + NSDocumentClass + BrowserDocument + + + CFBundleTypeExtensions + + pdf + + CFBundleTypeIconFile + pdf.icns + CFBundleTypeIconSystemGenerated + YES + CFBundleTypeMIMETypes + + application/pdf + + CFBundleTypeName + PDF document + CFBundleTypeRole + Viewer + LSHandlerRank + Alternate + NSDocumentClass + BrowserDocument + + + CFBundleTypeExtensions + + webarchive + + CFBundleTypeIconFile + webarchive.icns + CFBundleTypeIconSystemGenerated + YES + CFBundleTypeMIMETypes + + application/x-webarchive + + CFBundleTypeName + Web archive + CFBundleTypeRole + Viewer + ICExtension + ARCHIVE + LSHandlerRank + Default + LSIsAppleDefaultForType + + NSDocumentClass + BrowserDocument + + + CFBundleTypeExtensions + + webbookmark + + CFBundleTypeIconFile + webbookmark.icns + CFBundleTypeIconSystemGenerated + YES + CFBundleTypeName + Safari bookmark + CFBundleTypeRole + Viewer + LSHandlerRank + Default + NSDocumentClass + BrowserDocument + + + CFBundleTypeExtensions + + webhistory + + CFBundleTypeIconFile + webhistory.icns + CFBundleTypeIconSystemGenerated + YES + CFBundleTypeName + Safari history item + CFBundleTypeRole + Viewer + LSHandlerRank + Default + NSDocumentClass + BrowserDocument + + + CFBundleTypeExtensions + + webloc + + CFBundleTypeIconFile + webloc.icns + CFBundleTypeIconSystemGenerated + YES + CFBundleTypeName + Web internet location + CFBundleTypeOSTypes + + ilht + + CFBundleTypeRole + Viewer + LSHandlerRank + Default + NSDocumentClass + BrowserDocument + + + CFBundleTypeExtensions + + download + + CFBundleTypeIconFile + download10.icns + CFBundleTypeIconSystemGenerated + YES + CFBundleTypeName + Safari download + CFBundleTypeRole + Editor + LSHandlerRank + Default + LSTypeIsPackage + + NSDocumentClass + BrowserDocument + + + CFBundleTypeExtensions + + safariextz + + CFBundleTypeIconFile + safariextz.icns + CFBundleTypeIconSystemGenerated + YES + CFBundleTypeMIMETypes + + application/x-safari-extension + + CFBundleTypeName + Safari extension + CFBundleTypeRole + Viewer + LSHandlerRank + Owner + LSTypeIsPackage + + NSDocumentClass + BrowserDocument + + + CFBundleTypeExtensions + + gif + + CFBundleTypeIconFile + gif.icns + CFBundleTypeIconSystemGenerated + YES + CFBundleTypeMIMETypes + + image/gif + + CFBundleTypeName + GIF image + CFBundleTypeOSTypes + + GIFf + + CFBundleTypeRole + Viewer + LSHandlerRank + Alternate + NSDocumentClass + BrowserDocument + + + CFBundleTypeExtensions + + html + htm + shtml + jhtml + + CFBundleTypeIconFile + html.icns + CFBundleTypeIconSystemGenerated + YES + CFBundleTypeMIMETypes + + text/html + + CFBundleTypeName + HTML document + CFBundleTypeOSTypes + + HTML + + CFBundleTypeRole + Viewer + ICExtension + HTML + LSHandlerRank + Default + LSIsAppleDefaultForType + + NSDocumentClass + BrowserDocument + + + CFBundleTypeExtensions + + js + + CFBundleTypeIconFile + js.icns + CFBundleTypeIconSystemGenerated + YES + CFBundleTypeMIMETypes + + application/x-javascript + + CFBundleTypeName + JavaScript script + CFBundleTypeRole + Viewer + LSHandlerRank + Alternate + NSDocumentClass + BrowserDocument + + + CFBundleTypeExtensions + + jpg + jpeg + + CFBundleTypeIconFile + jpeg.icns + CFBundleTypeIconSystemGenerated + YES + CFBundleTypeMIMETypes + + image/jpeg + + CFBundleTypeName + JPEG image + CFBundleTypeOSTypes + + JPEG + + CFBundleTypeRole + Viewer + ICExtension + JPEG + LSHandlerRank + Alternate + NSDocumentClass + BrowserDocument + + + CFBundleTypeExtensions + + jp2 + + CFBundleTypeIconFile + jp2.icns + CFBundleTypeIconSystemGenerated + YES + CFBundleTypeMIMETypes + + image/jp2 + + CFBundleTypeName + JPEG 2000 image + CFBundleTypeOSTypes + + jp2 + + CFBundleTypeRole + Viewer + LSHandlerRank + Alternate + NSDocumentClass + BrowserDocument + + + CFBundleTypeExtensions + + txt + text + + CFBundleTypeIconFile + txt.icns + CFBundleTypeIconSystemGenerated + YES + CFBundleTypeMIMETypes + + text/plain + + CFBundleTypeName + Plain text document + CFBundleTypeOSTypes + + TEXT + + CFBundleTypeRole + Viewer + ICExtension + TXT + LSHandlerRank + Alternate + NSDocumentClass + BrowserDocument + + + CFBundleTypeExtensions + + png + + CFBundleTypeIconFile + png.icns + CFBundleTypeMIMETypes + + image/png + + CFBundleTypeName + PNG image + CFBundleTypeOSTypes + + PNGf + + CFBundleTypeRole + Viewer + LSHandlerRank + Alternate + NSDocumentClass + BrowserDocument + + + CFBundleTypeExtensions + + tiff + tif + + CFBundleTypeIconFile + tiff.icns + CFBundleTypeIconSystemGenerated + YES + CFBundleTypeMIMETypes + + image/tiff + + CFBundleTypeName + TIFF image + CFBundleTypeOSTypes + + TIFF + + CFBundleTypeRole + Viewer + ICExtension + TIFF + LSHandlerRank + Alternate + NSDocumentClass + BrowserDocument + + + CFBundleTypeExtensions + + url + + CFBundleTypeIconFile + url.icns + CFBundleTypeIconSystemGenerated + YES + CFBundleTypeName + Web site location + CFBundleTypeOSTypes + + LINK + + CFBundleTypeRole + Viewer + LSHandlerRank + Default + LSIsAppleDefaultForType + + NSDocumentClass + BrowserDocument + + + CFBundleTypeExtensions + + ico + + CFBundleTypeIconFile + ico.icns + CFBundleTypeIconSystemGenerated + YES + CFBundleTypeMIMETypes + + image/x-icon + + CFBundleTypeName + Windows icon image + CFBundleTypeOSTypes + + ICO + + CFBundleTypeRole + Viewer + LSHandlerRank + Alternate + NSDocumentClass + BrowserDocument + + + CFBundleTypeExtensions + + xhtml + xht + xhtm + xht + + CFBundleTypeIconFile + xhtml.icns + CFBundleTypeIconSystemGenerated + YES + CFBundleTypeMIMETypes + + application/xhtml+xml + + CFBundleTypeName + XHTML document + CFBundleTypeRole + Viewer + ICExtension + XHTML + LSHandlerRank + Default + NSDocumentClass + BrowserDocument + + + CFBundleTypeExtensions + + xml + xbl + xsl + xslt + + CFBundleTypeIconFile + xml.icns + CFBundleTypeIconSystemGenerated + YES + CFBundleTypeMIMETypes + + application/xml + text/xml + + CFBundleTypeName + XML document + CFBundleTypeRole + Viewer + ICExtension + XML + LSHandlerRank + Default + NSDocumentClass + BrowserDocument + + + CFBundleTypeExtensions + + svg + + CFBundleTypeIconFile + svg.icns + CFBundleTypeIconSystemGenerated + YES + CFBundleTypeMIMETypes + + image/svg+xml + + CFBundleTypeName + SVG document + CFBundleTypeRole + Viewer + LSHandlerRank + Alternate + NSDocumentClass + BrowserDocument + + + CFBundleExecutable + Safari + CFBundleGetInfoString + 15.6, Copyright © 2003-2022 Apple Inc. + CFBundleHelpBookFolder + Safari.help + CFBundleHelpBookName + com.apple.Safari.help + CFBundleIconFile + AppIcon + CFBundleIconName + AppIcon + CFBundleIdentifier + com.apple.Safari + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + Safari + CFBundlePackageType + APPL + CFBundleShortVersionString + 15.6 + CFBundleSignature + sfri + CFBundleSupportedPlatforms + + MacOSX + + CFBundleURLTypes + + + CFBundleURLName + Web site URL + CFBundleURLSchemes + + http + https + + LSHandlerRank + Default + LSIsAppleDefaultForScheme + + + + CFBundleURLName + Local file URL + CFBundleURLSchemes + + file + + LSHandlerRank + Default + + + CFBundleURLSchemes + + prefs + + LSHandlerRank + None + + + CFBundleVersion + 17613.3.9.1.5 + DTCompiler + com.apple.compilers.llvm.clang.1_0 + DTPlatformBuild + 21G66 + DTPlatformName + macosx + DTPlatformVersion + 12.5 + DTSDKBuild + 21G66 + DTSDKName + macosx12.5.internal + DTXcode + 1330 + DTXcodeBuild + 13E6049a + HPDHelpProjectIdentifier + safari + LSApplicationCategoryType + public.app-category.productivity + LSFileQuarantineEnabled + + LSMinimumSystemVersion + 12.5.0 + NSAppleScriptEnabled + Yes + NSExtensionSDK + + com.apple.Safari.web-extension + + NSExtension + + NSExtensionAttributes + + NSExtensionPointName + com.apple.Safari.web-extension + NSExtensionPointVersion + 1.0 + + NSExtensionHostEntitlement + com.apple.private.can-load-any-content-blocker + NSExtensionProtocol + NSObject + PrincipalClass + NSObject + Subsystems + + NSSharingService_Subsystem + + + XPCService + + JoinExistingSession + + RunLoopType + NSRunLoop + ServiceType + Application + _AdditionalSubServices + + apple-extension-service + + + + + + NSLocationUsageDescription + Websites you visit may request your location. + NSMainNibFile + MainMenu + NSPrincipalClass + BrowserApplication + NSServices + + + NSKeyEquivalent + + default + L + + NSMenuItem + + default + Search With %WebSearchProvider@ + + NSMessage + searchWithWebSearchProvider + NSPortName + Safari + NSSendTypes + + public.utf8-plain-text + + + + NSMenuItem + + default + Add to Reading List + + NSMessage + addToReadingList + NSPortName + Safari + NSRequiredContext + + + NSTextContent + URL + + + NSLinkSchemes + + http + https + + + + NSSendTypes + + public.rtf + public.utf8-plain-text + + + + NSSpeechRecognitionUsageDescription + Websites you visit may access speech recognition. + NSSupportsAutomaticTermination + + NSSupportsSuddenTermination + + NSUserActivityTypes + + NSUserActivityTypeBrowsingWeb + + OSAScriptingDefinition + Safari.sdef + UTExportedTypeDeclarations + + + UTTypeConformsTo + + public.data + + UTTypeDescription + Safari bookmark + UTTypeIdentifier + com.apple.safari.bookmark + UTTypeTagSpecification + + public.filename-extension + + webbookmark + + + + + UTTypeConformsTo + + public.data + + UTTypeDescription + Safari extension + UTTypeIdentifier + com.apple.safari.extension + UTTypeTagSpecification + + public.filename-extension + + safariextz + + + + + UTTypeConformsTo + + public.data + + UTTypeDescription + Safari history item + UTTypeIdentifier + com.apple.safari.history + UTTypeTagSpecification + + public.filename-extension + + webhistory + + + + + + From 47447577264d3c8386d90c56cca48b75671e2717 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sun, 31 Jul 2022 16:47:48 -0700 Subject: [PATCH 034/116] fixup for byte and datetime values --- jc/parsers/plist.py | 58 +++++++++++++++++++++ tests/fixtures/generic/plist-alltypes.plist | 49 +++++++++++++++++ 2 files changed, 107 insertions(+) create mode 100644 tests/fixtures/generic/plist-alltypes.plist diff --git a/jc/parsers/plist.py b/jc/parsers/plist.py index 3578ac590..ae3119fe8 100644 --- a/jc/parsers/plist.py +++ b/jc/parsers/plist.py @@ -2,6 +2,10 @@ Converts binary and XML PLIST files. +Binary values are converted into an ASCII hex representation. + +Datetime objects are converted into Unix epoch timestamps and ISO strings. + Usage (cli): $ cat file.plist | jc --plist @@ -31,6 +35,8 @@ """ from typing import List, Dict, Union import plistlib +import binascii +from datetime import datetime import jc.utils @@ -61,6 +67,57 @@ def _process(proc_data: Dict) -> Dict: return proc_data +def _b2a(byte_string: bytes) -> str: + """Convert a byte string to a colon-delimited hex ascii string""" + # need try/except since seperator was only introduced in python 3.8. + # provides compatibility for python 3.6 and 3.7. + try: + return binascii.hexlify(byte_string, ':').decode('utf-8') + except TypeError: + hex_string = binascii.hexlify(byte_string).decode('utf-8') + colon_seperated = ':'.join(hex_string[i:i+2] for i in range(0, len(hex_string), 2)) + return colon_seperated + + +def _fix_objects(obj): + """ + Recursively traverse the nested dictionary or list and convert objects + into JSON serializable types. + """ + if isinstance(obj, dict): + for k, v in obj.copy().items(): + + if isinstance(v, datetime): + iso = v.isoformat() + v = int(round(v.timestamp())) + obj.update({k: v, f'{k}_iso': iso}) + continue + + if isinstance(v, bytes): + v = _b2a(v) + obj.update({k: v}) + continue + + if isinstance(v, dict): + obj.update({k: _fix_objects(v)}) + continue + + if isinstance(v, list): + newlist =[] + for i in v: + newlist.append(_fix_objects(i)) + obj.update({k: newlist}) + continue + + if isinstance(obj, list): + new_list = [] + for i in obj: + new_list.append(_fix_objects(i)) + obj = new_list + + return obj + + def parse( data: Union[str, bytes], raw: bool = False, @@ -89,5 +146,6 @@ def parse( if jc.utils.has_data(data): raw_output = plistlib.loads(data) + raw_output = _fix_objects(raw_output) return raw_output if raw else _process(raw_output) diff --git a/tests/fixtures/generic/plist-alltypes.plist b/tests/fixtures/generic/plist-alltypes.plist new file mode 100644 index 000000000..fccd690e3 --- /dev/null +++ b/tests/fixtures/generic/plist-alltypes.plist @@ -0,0 +1,49 @@ + + + + + aDate + 2022-07-31T16:34:30Z + aDict + + aFalseValue + + aThirdString + Mässig, Maß + aTrueValue + + anotherString + <hello & hi there!> + + aFloat + 0.1 + aList + + A + B + 12 + 32.1 + + 1 + 2 + 3 + + + aString + Doodah + anInt + 728 + someData + + PGJpbmFyeSBndW5rPg== + + someMoreData + + PGxvdHMgb2YgYmluYXJ5IGd1bms+PGxvdHMgb2YgYmluYXJ5IGd1bms+PGxvdHMgb2Yg + YmluYXJ5IGd1bms+PGxvdHMgb2YgYmluYXJ5IGd1bms+PGxvdHMgb2YgYmluYXJ5IGd1 + bms+PGxvdHMgb2YgYmluYXJ5IGd1bms+PGxvdHMgb2YgYmluYXJ5IGd1bms+PGxvdHMg + b2YgYmluYXJ5IGd1bms+PGxvdHMgb2YgYmluYXJ5IGd1bms+PGxvdHMgb2YgYmluYXJ5 + IGd1bms+ + + + From 86067f913b11da7ad7a0211e216f69b5df535037 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sun, 31 Jul 2022 17:08:44 -0700 Subject: [PATCH 035/116] fix for unparsable bytes --- jc/parsers/plist.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/jc/parsers/plist.py b/jc/parsers/plist.py index ae3119fe8..64201b0fe 100644 --- a/jc/parsers/plist.py +++ b/jc/parsers/plist.py @@ -145,7 +145,11 @@ def parse( if jc.utils.has_data(data): - raw_output = plistlib.loads(data) + try: + raw_output = plistlib.loads(data) + except plistlib.InvalidFileException: + pass + raw_output = _fix_objects(raw_output) return raw_output if raw else _process(raw_output) From f22ac8db2b0fbd134a4137156cbb3e40367e1bb4 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sun, 31 Jul 2022 17:16:17 -0700 Subject: [PATCH 036/116] fix for empty string data --- jc/parsers/plist.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/jc/parsers/plist.py b/jc/parsers/plist.py index 64201b0fe..bf8122154 100644 --- a/jc/parsers/plist.py +++ b/jc/parsers/plist.py @@ -138,18 +138,14 @@ def parse( """ jc.utils.compatibility(__name__, info.compatible, quiet) - if isinstance(data, str): - data = bytes(data, 'utf-8') - raw_output: Dict = {} if jc.utils.has_data(data): - try: - raw_output = plistlib.loads(data) - except plistlib.InvalidFileException: - pass + if isinstance(data, str): + data = bytes(data, 'utf-8') + raw_output = plistlib.loads(data) raw_output = _fix_objects(raw_output) return raw_output if raw else _process(raw_output) From 0bfb5f8a3c6e09b37d7ab6ccc0a98a966d626668 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sun, 31 Jul 2022 17:19:04 -0700 Subject: [PATCH 037/116] add FortiSOAR --- README.md | 1 + templates/readme_template | 1 + 2 files changed, 2 insertions(+) diff --git a/README.md b/README.md index 21f23ba6f..ac9a7c26f 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,7 @@ Use Cases: - [Ansible command output parsing](https://blog.kellybrazil.com/2020/08/30/parsing-command-output-in-ansible-with-jc/) - [Saltstack command output parsing](https://blog.kellybrazil.com/2020/09/15/parsing-command-output-in-saltstack-with-jc/) - [Nornir command output parsing](https://blog.kellybrazil.com/2020/12/09/parsing-command-output-in-nornir-with-jc/) +- [FortiSOAR command output parsing](https://docs.fortinet.com/document/fortisoar/1.0.0/jc-parse-command-output/323/jc-parse-command-output-v1-0-0) ## Installation There are several ways to get `jc`. You can install via `pip`, OS package diff --git a/templates/readme_template b/templates/readme_template index 61fa32697..147bb6909 100644 --- a/templates/readme_template +++ b/templates/readme_template @@ -93,6 +93,7 @@ Use Cases: - [Ansible command output parsing](https://blog.kellybrazil.com/2020/08/30/parsing-command-output-in-ansible-with-jc/) - [Saltstack command output parsing](https://blog.kellybrazil.com/2020/09/15/parsing-command-output-in-saltstack-with-jc/) - [Nornir command output parsing](https://blog.kellybrazil.com/2020/12/09/parsing-command-output-in-nornir-with-jc/) +- [FortiSOAR command output parsing](https://docs.fortinet.com/document/fortisoar/1.0.0/jc-parse-command-output/323/jc-parse-command-output-v1-0-0) ## Installation There are several ways to get `jc`. You can install via `pip`, OS package From 02f7d73fcab3b1053852ec610d723fbcca52d154 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sun, 31 Jul 2022 17:21:50 -0700 Subject: [PATCH 038/116] add fortisoar --- README.md | 1 + templates/readme_template | 1 + 2 files changed, 2 insertions(+) diff --git a/README.md b/README.md index ac9a7c26f..278f64207 100644 --- a/README.md +++ b/README.md @@ -121,6 +121,7 @@ pip3 install jc | macOS | `brew install jc` | | FreeBSD | `portsnap fetch update && cd /usr/ports/textproc/py-jc && make install clean` | | Ansible filter plugin | `ansible-galaxy collection install community.general` | +| FortiSOAR connector | Install from FortiSOAR Connector Marketplace | > For more OS Packages, see https://repology.org/project/jc/versions. diff --git a/templates/readme_template b/templates/readme_template index 147bb6909..27bb1e5f0 100644 --- a/templates/readme_template +++ b/templates/readme_template @@ -121,6 +121,7 @@ pip3 install jc | macOS | `brew install jc` | | FreeBSD | `portsnap fetch update && cd /usr/ports/textproc/py-jc && make install clean` | | Ansible filter plugin | `ansible-galaxy collection install community.general` | +| FortiSOAR connector | Install from FortiSOAR Connector Marketplace | > For more OS Packages, see https://repology.org/project/jc/versions. From 63961d871181600cda102f78aec365ad5a16dcdb Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Mon, 1 Aug 2022 11:11:26 -0700 Subject: [PATCH 039/116] add plist tests --- .../fixtures/generic/plist-alltypes-bin.json | 1 + .../fixtures/generic/plist-alltypes-bin.plist | Bin 0 -> 576 bytes tests/fixtures/generic/plist-alltypes.json | 1 + .../generic/plist-garageband-info.json | 1 + tests/fixtures/generic/plist-safari-info.json | 1 + tests/test_plist.py | 75 ++++++++++++++++++ 6 files changed, 79 insertions(+) create mode 100644 tests/fixtures/generic/plist-alltypes-bin.json create mode 100644 tests/fixtures/generic/plist-alltypes-bin.plist create mode 100644 tests/fixtures/generic/plist-alltypes.json create mode 100644 tests/fixtures/generic/plist-garageband-info.json create mode 100644 tests/fixtures/generic/plist-safari-info.json create mode 100644 tests/test_plist.py diff --git a/tests/fixtures/generic/plist-alltypes-bin.json b/tests/fixtures/generic/plist-alltypes-bin.json new file mode 100644 index 000000000..bdbe76b1e --- /dev/null +++ b/tests/fixtures/generic/plist-alltypes-bin.json @@ -0,0 +1 @@ +{"aDate":1659369751,"aDict":{"aFalseValue":false,"aThirdString":"Mässig, Maß","aTrueValue":true,"anotherString":""},"aFloat":0.1,"aList":["A","B",12,32.1,[1,2,3]],"aString":"Doodah","anInt":728,"someData":"3c:62:69:6e:61:72:79:20:67:75:6e:6b:3e","someMoreData":"3c:6c:6f:74:73:20:6f:66:20:62:69:6e:61:72:79:20:67:75:6e:6b:3e:3c:6c:6f:74:73:20:6f:66:20:62:69:6e:61:72:79:20:67:75:6e:6b:3e:3c:6c:6f:74:73:20:6f:66:20:62:69:6e:61:72:79:20:67:75:6e:6b:3e:3c:6c:6f:74:73:20:6f:66:20:62:69:6e:61:72:79:20:67:75:6e:6b:3e:3c:6c:6f:74:73:20:6f:66:20:62:69:6e:61:72:79:20:67:75:6e:6b:3e:3c:6c:6f:74:73:20:6f:66:20:62:69:6e:61:72:79:20:67:75:6e:6b:3e:3c:6c:6f:74:73:20:6f:66:20:62:69:6e:61:72:79:20:67:75:6e:6b:3e:3c:6c:6f:74:73:20:6f:66:20:62:69:6e:61:72:79:20:67:75:6e:6b:3e:3c:6c:6f:74:73:20:6f:66:20:62:69:6e:61:72:79:20:67:75:6e:6b:3e:3c:6c:6f:74:73:20:6f:66:20:62:69:6e:61:72:79:20:67:75:6e:6b:3e","aDate_iso":"2022-08-01T09:02:31.181281"} diff --git a/tests/fixtures/generic/plist-alltypes-bin.plist b/tests/fixtures/generic/plist-alltypes-bin.plist new file mode 100644 index 0000000000000000000000000000000000000000..c1ab17e874250d2dfe727fe2286ee69be8846f81 GIT binary patch literal 576 zcmYc)$jK}&F)+Bn$i&RT%Er#Y$t5fzD<`iIn&^^Pk_sdbK&VG=2y6Rc=`AR1cjm#-4b(( zQ^OK-N>gJJLozapQXs}eC59B0Ld0Ve^YTkFQi~uG9N7%M3{Mz}fiROHok52|0mx5e zxX&3cAZ(M7nv;{Spr(+KsQ}WEs%WQdzjG!C%vvfcCN3cv=osiEz@zNoz;gx&&Muac zmJwhSU=m;sbIH$7Nz4#ry5Vb+l$n=URH=|&nwM?oFL22wC%>dvAwNw4S$s$fGH@_R zGRQNi0z=P;!HU6w!IL43A)X!)%6444WBFGF)P~!ElS=4#Pc$=M1kI e-ZH#r_{i{s;Wr~E0~jzdLTCm>C=I0;?*RbcBe#_R literal 0 HcmV?d00001 diff --git a/tests/fixtures/generic/plist-alltypes.json b/tests/fixtures/generic/plist-alltypes.json new file mode 100644 index 000000000..55ef893b0 --- /dev/null +++ b/tests/fixtures/generic/plist-alltypes.json @@ -0,0 +1 @@ +{"aDate":1659310470,"aDict":{"aFalseValue":false,"aThirdString":"Mässig, Maß","aTrueValue":true,"anotherString":""},"aFloat":0.1,"aList":["A","B",12,32.1,[1,2,3]],"aString":"Doodah","anInt":728,"someData":"3c:62:69:6e:61:72:79:20:67:75:6e:6b:3e","someMoreData":"3c:6c:6f:74:73:20:6f:66:20:62:69:6e:61:72:79:20:67:75:6e:6b:3e:3c:6c:6f:74:73:20:6f:66:20:62:69:6e:61:72:79:20:67:75:6e:6b:3e:3c:6c:6f:74:73:20:6f:66:20:62:69:6e:61:72:79:20:67:75:6e:6b:3e:3c:6c:6f:74:73:20:6f:66:20:62:69:6e:61:72:79:20:67:75:6e:6b:3e:3c:6c:6f:74:73:20:6f:66:20:62:69:6e:61:72:79:20:67:75:6e:6b:3e:3c:6c:6f:74:73:20:6f:66:20:62:69:6e:61:72:79:20:67:75:6e:6b:3e:3c:6c:6f:74:73:20:6f:66:20:62:69:6e:61:72:79:20:67:75:6e:6b:3e:3c:6c:6f:74:73:20:6f:66:20:62:69:6e:61:72:79:20:67:75:6e:6b:3e:3c:6c:6f:74:73:20:6f:66:20:62:69:6e:61:72:79:20:67:75:6e:6b:3e:3c:6c:6f:74:73:20:6f:66:20:62:69:6e:61:72:79:20:67:75:6e:6b:3e","aDate_iso":"2022-07-31T16:34:30"} diff --git a/tests/fixtures/generic/plist-garageband-info.json b/tests/fixtures/generic/plist-garageband-info.json new file mode 100644 index 000000000..c80d229e6 --- /dev/null +++ b/tests/fixtures/generic/plist-garageband-info.json @@ -0,0 +1 @@ +{"NSAppleScriptEnabled":true,"LSMultipleInstancesProhibited":true,"CFBundleInfoDictionaryVersion":"6.0","DTPlatformVersion":"GM","CFBundleIconFile":"GarageBand.icns","CFBundleName":"GarageBand","DTSDKName":"macosx10.13internal","NSSupportsAutomaticGraphicsSwitching":true,"RevisionDate":"2018-12-03_14:10:56","UTImportedTypeDeclarations":[{"UTTypeConformsTo":["public.data","public.content"],"UTTypeIconFile":"EXS24Instrument.icns","UTTypeIdentifier":"com.apple.logic.exs","UTTypeDescription":"EXS instrument","UTTypeTagSpecification":{"com.apple.ostype":[".exs",".SBK"],"public.filename-extension":["exs","sbk"]}},{"UTTypeConformsTo":["public.data","public.content"],"UTTypeIconFile":"LogicPluginSetting.icns","UTTypeIdentifier":"com.apple.logic.pluginsetting","UTTypeDescription":"Logic Audio Plug-In Setting proprietary format","UTTypeTagSpecification":{"com.apple.ostype":"EM9S","public.filename-extension":"pst"}},{"UTTypeConformsTo":["public.data","public.audiovisual-content"],"UTTypeIdentifier":"com.apple.logic.notator","UTTypeDescription":"Notator SL Song","UTTypeTagSpecification":{"public.filename-extension":"SON"}},{"UTTypeConformsTo":["public.data","public.content","com.apple.package"],"UTTypeIdentifier":"com.apple.logic.gchdb","UTTypeDescription":"Guitar Chord Library","UTTypeTagSpecification":{"public.filename-extension":"gchdb"}},{"UTTypeConformsTo":["public.data","public.content","com.apple.package"],"UTTypeIconFile":"LogicXProject.icns","UTTypeIdentifier":"com.apple.logicx.project","UTTypeDescription":"Logic X Project","UTTypeTagSpecification":{"public.filename-extension":["logicx"]}},{"UTTypeConformsTo":["public.data","public.content"],"UTTypeIconFile":"ImpulseResponse.icns","UTTypeIdentifier":"com.apple.Logic.SpaceDesignerImpulseResponse","UTTypeDescription":"Space Designer Impulse Responses","UTTypeTagSpecification":{"com.apple.ostype":["SDIR"],"public.mime-type":[],"public.filename-extension":["sdir","SDIR"]}},{"UTTypeConformsTo":["public.data","public.content","com.apple.package"],"UTTypeIconFile":"","UTTypeIdentifier":"com.apple.logic.drummerlibrary","UTTypeDescription":"Drummer Library","UTTypeTagSpecification":{"public.filename-extension":["glibb"]}},{"UTTypeConformsTo":["public.data","public.content"],"UTTypeIconFile":"DPST.icns","UTTypeIdentifier":"com.apple.logic.dpst","UTTypeDescription":"Drummer Editor Preset","UTTypeTagSpecification":{"public.filename-extension":["dpst"]}},{"UTTypeConformsTo":["public.data"],"UTTypeIdentifier":"com.apple.mainstage.graph","UTTypeDescription":"MainStage Graph Preset","UTTypeTagSpecification":{"public.filename-extension":["mainstageGraph"]}},{"UTTypeConformsTo":["public.data","public.content"],"UTTypeIconFile":"LogicKeyCommands","UTTypeIdentifier":"com.apple.logic.keycommand","UTTypeDescription":"Logic Key Commands proprietary format","UTTypeTagSpecification":{"com.apple.ostype":"LkcS","public.filename-extension":"logikcs"}},{"UTTypeConformsTo":["com.apple.localizable-name-bundle","com.apple.package"],"UTTypeIconFile":"Patch.icns","UTTypeIdentifier":"com.apple.mainstage.patches","UTTypeDescription":"MainStage Patch","UTTypeTagSpecification":{"public.filename-extension":["patch"]}},{"UTTypeConformsTo":["public.xml","public.data"],"UTTypeIdentifier":"com.apple.audio-unit-preset","UTTypeDescription":"audio unit preset","UTTypeTagSpecification":{"public.filename-extension":["aupreset"]}},{"UTTypeConformsTo":["public.data","public.composite-content"],"UTTypeReferenceURL":"http://www.aafassociation.org/html/techinfo/index.html","UTTypeIdentifier":"org.aafassociation.advanced-authoring-format","UTTypeDescription":"Advanced Authoring Format","UTTypeTagSpecification":{"com.apple.ostype":["AAF "],"public.filename-extension":["aaf"]}},{"UTTypeConformsTo":"public.audiovisual-content","UTTypeIdentifier":"com.avid.open-media-framework","UTTypeDescription":"Open Media Framework interchange format","UTTypeTagSpecification":{"com.apple.ostype":["OMF ","OMFI"],"public.filename-extension":["omf","omfi"]}},{"UTTypeConformsTo":["public.data","public.composite-content"],"UTTypeIdentifier":"com.tascam.opentl","UTTypeDescription":"Open TL format","UTTypeTagSpecification":{"com.apple.ostype":["OPTL"],"public.filename-extension":["tl"]}},{"UTTypeConformsTo":"public.audio","UTTypeIdentifier":"public.midi-audio","UTTypeDescription":"MIDI audio","UTTypeTagSpecification":{"com.apple.ostype":["Midi"],"public.mime-type":["audio/midi","audio/x-midi"],"public.filename-extension":["midi","mid","smf","kar"]}},{"UTTypeConformsTo":["public.data","public.audiovisual-content"],"UTTypeIconFile":"LogicSong.icns","UTTypeIdentifier":"com.apple.logic-song","UTTypeDescription":"Logic Song","UTTypeTagSpecification":{"com.apple.ostype":["EM3F","sM3F"],"public.filename-extension":"lso"}},{"UTTypeConformsTo":["public.data","public.content","com.apple.package"],"UTTypeIconFile":"LogicProject.icns","UTTypeIdentifier":"com.apple.logic.project","UTTypeDescription":"Logic Project","UTTypeTagSpecification":{"com.apple.ostype":"????","public.filename-extension":"logic"}},{"UTTypeConformsTo":["public.data","public.content"],"UTTypeIconFile":"LogicCST.icns","UTTypeIdentifier":"com.apple.logic.channelstripsetting","UTTypeDescription":"Logic Audio Channel Strip Setting proprietary format","UTTypeTagSpecification":{"com.apple.ostype":".CST","public.filename-extension":"cst"}}],"NSPrincipalClass":"CLgApplication","LSApplicationCategoryType":"public.app-category.music","CFBundleDisplayName":"GarageBand","CFBundleShortName":"GarageBand","CFBundleDocumentTypes":[{"CFBundleTypeExtensions":["band","gbProj"],"CFBundleTypeOSTypes":["EMP1"],"LSTypeIsPackage":true,"NSPersistentStoreTypeKey":"Binary","CFBundleTypeName":"GarageBand Project","CFBundleTypeIconFile":"GarageBandDocument","LSItemContentTypes":["com.apple.garageband.project"],"CFBundleTypeRole":"Editor","NSDocumentClass":"CLgSongDocument"},{"CFBundleTypeExtensions":["mwand"],"LSTypeIsPackage":true,"NSPersistentStoreTypeKey":"Binary","CFBundleTypeName":"GarageBandMagicMentorTemplate","CFBundleTypeIconFile":"GarageBandDocument.icns","LSItemContentTypes":["com.apple.garageband.template.magicmentor"],"CFBundleTypeRole":"Editor","NSDocumentClass":"DfDocument"},{"CFBundleTypeExtensions":["patch"],"LSTypeIsPackage":true,"CFBundleTypeName":"Logic X Patch","CFBundleTypeIconFile":"Patch.icns","LSItemContentTypes":["com.apple.mainstage.patches"],"CFBundleTypeRole":"Viewer","NSDocumentClass":"CLgSongDocument"},{"CFBundleTypeExtensions":["mid"],"CFBundleTypeOSTypes":["Midi"],"LSTypeIsPackage":false,"CFBundleTypeName":"Standard MIDI File","CFBundleTypeIconFile":"StandardMIDIFile.icns","LSItemContentTypes":["public.midi-audio"],"CFBundleTypeRole":"Viewer","NSDocumentClass":"CLgSongDocument"},{"CFBundleTypeExtensions":["aif","aiff"],"CFBundleTypeOSTypes":["AIFF"],"LSTypeIsPackage":false,"CFBundleTypeName":"AIFF Audio file","CFBundleTypeIconFile":"LogicAudioFileAIFF.icns","LSItemContentTypes":["public.aiff-audio"],"CFBundleTypeRole":"Editor"},{"CFBundleTypeExtensions":["caf"],"LSTypeIsPackage":false,"NSPersistentStoreTypeKey":"Binary","CFBundleTypeName":"CAF Audio file","CFBundleTypeIconFile":"LogicAudioFileCAF.icns","LSItemContentTypes":["com.apple.coreaudio-format"],"CFBundleTypeRole":"Editor"},{"CFBundleTypeExtensions":["sd2"],"CFBundleTypeOSTypes":["Sd2f"],"LSTypeIsPackage":false,"CFBundleTypeName":"SoundDesigner II audio file","CFBundleTypeIconFile":"LogicAudioFileSDII.icns","LSItemContentTypes":["com.digidesign.sd2-audio"],"CFBundleTypeRole":"Editor"},{"CFBundleTypeExtensions":["wav","wave"],"CFBundleTypeOSTypes":["WAVE"],"LSTypeIsPackage":false,"CFBundleTypeName":"Wave file","CFBundleTypeIconFile":"LogicAudioFileWAV.icns","LSItemContentTypes":["com.microsoft.waveform-audio"],"CFBundleTypeRole":"Editor"},{"CFBundleTypeExtensions":["pst"],"CFBundleTypeOSTypes":["PSET","EM9S"],"LSTypeIsPackage":false,"CFBundleTypeName":"Logic Plug-in settings file","CFBundleTypeIconFile":"LogicPluginSetting.icns","LSItemContentTypes":["com.apple.logic.pluginsetting"],"CFBundleTypeRole":"Editor"},{"CFBundleTypeExtensions":["cst"],"LSTypeIsPackage":false,"NSPersistentStoreTypeKey":"Binary","CFBundleTypeName":"Logic Channel Strip settings file","CFBundleTypeIconFile":"LogicCST.icns","LSItemContentTypes":["com.apple.logic.channelstripsetting"],"CFBundleTypeRole":"Editor"},{"CFBundleTypeOSTypes":["LABL"],"LSTypeIsPackage":false,"CFBundleTypeName":"Logic I/O label file","CFBundleTypeIconFile":"LogicIOLabel.icns","LSItemContentTypes":["com.apple.logic.iolabels"],"CFBundleTypeRole":"Editor"},{"CFBundleTypeExtensions":["sdir","SDIR"],"CFBundleTypeOSTypes":["SDIR","AIFF"],"LSTypeIsPackage":false,"CFBundleTypeName":"Space Designer Impulse Response","CFBundleTypeIconFile":"ImpulseResponse.icns","LSItemContentTypes":["com.apple.Logic.SpaceDesignerImpulseResponse"],"CFBundleTypeRole":"None"},{"CFBundleTypeExtensions":["exs","SBK"],"CFBundleTypeOSTypes":[".exs",".SBK"],"LSTypeIsPackage":false,"CFBundleTypeName":"EXS24 instrument file","CFBundleTypeIconFile":"EXS24Instrument.icns","LSItemContentTypes":["com.apple.logic.exs"],"CFBundleTypeRole":"Editor"},{"CFBundleTypeExtensions":["logikcs"],"LSTypeIsPackage":false,"NSPersistentStoreTypeKey":"XML","CFBundleTypeName":"Logic Key Commands","CFBundleTypeIconFile":"LogicKeyCommands.icns","LSItemContentTypes":["com.apple.logic.keycommand"],"CFBundleTypeRole":"None"},{"CFBundleTypeExtensions":["gchdb"],"LSTypeIsPackage":true,"NSPersistentStoreTypeKey":"Binary","CFBundleTypeName":"Guitar Chord Library","LSItemContentTypes":["com.apple.logic.gchdb"],"CFBundleTypeRole":"Editor"},{"CFBundleTypeExtensions":["dpst"],"CFBundleTypeName":"Drummer Editor Preset","CFBundleTypeIconFile":"DPST.icns","LSItemContentTypes":["com.apple.logic.dpst"],"CFBundleTypeRole":"None"}],"DTSDKBuild":"17D47","MADefaultResourceBundleID":"com.apple.music.apps.MAResources","CFBundleShortVersionString":"10.3.2","CFBundleSupportedPlatforms":["MacOSX"],"BuildMachineOSBuild":"16B2657","BugComponent":"GarageBand MacOS | New Bugs","CFBundlePackageType":"APPL","DTXcodeBuild":"9C40b","CFBundleDevelopmentRegion":"English","DTPlatformBuild":"9C40b","CFBundleVersion":"4980.96","MAApplicationResourceBundleID":"com.apple.music.apps.MAResourcesGB","ATSApplicationFontsPath":"../Frameworks/MAResources.framework/Resources","NSUbiquitousContainers":{"F3LWYJ7GM7.com.apple.garageband10":{"NSUbiquitousContainerIsDocumentScopePublic":true,"NSUbiquitousContainerSupportedFolderLevels":"One","NSUbiquitousContainerName":"GarageBand for macOS"},"F3LWYJ7GM7.com.apple.mobilegarageband":{"NSUbiquitousContainerIsDocumentScopePublic":true,"NSUbiquitousContainerSupportedFolderLevels":"One","NSUbiquitousContainerName":"GarageBand for iOS"},"F3LWYJ7GM7.com.apple.musicmemos.ideas":{"NSUbiquitousContainerIsDocumentScopePublic":true,"NSUbiquitousContainerSupportedFolderLevels":"One","NSUbiquitousContainerName":"Music Memos"}},"CFBundleGetInfoString":"GarageBand 10.3.2, Copyright © 2004–2018 Apple Inc. All Rights Reserved","RevisionTrain":"MAManchester","UTExportedTypeDeclarations":[{"UTTypeConformsTo":["com.apple.package","public.audiovisual-content"],"UTTypeIconFile":"GarageBandDocument","UTTypeIdentifier":"com.apple.garageband.project","UTTypeDescription":"GarageBand Project","UTTypeReferenceURL":"http://www.apple.com/ilife/garageband/","UTTypeTagSpecification":{"public.filename-extension":["band","gbProj"]}},{"UTTypeConformsTo":["com.apple.package","public.audiovisual-content"],"UTTypeIconFile":"GarageBandDocument.icns","UTTypeIdentifier":"com.apple.garageband.template.magicmentor","UTTypeDescription":"MagicMentor GarageBand Template","UTTypeReferenceURL":"http://www.apple.com/ilife/garageband/","UTTypeTagSpecification":{"public.filename-extension":["mwand"]}}],"OSAScriptingDefinition":"GarageBandSuite.sdef","NSDisablePersistence":true,"NSMainNibFile":"MainMenu","NSMicrophoneUsageDescription":"To enable audio input for recording in GarageBand, click OK (recommended). You can also enable audio input later in System Preferences > Security & Privacy.","CFBundleIdentifier":"com.apple.garageband10","DTXcode":"0920","CFBundleHelpBookName":"com.apple.garageband10.help","CFBundleHelpBookFolder":"garageband.help","WhatsNewPanelVersion":3,"CFBundleExecutable":"GarageBand","LSMinimumSystemVersion":"10.12","CFBundleSignature":"band","RevisionBranch":"94d3b910e514cb711d268bc5d953046c3bb629e70a5564a403b3bde53d2a75b7","RevisionInfo":"b5a3763eeeb9cd03d024bdb84b9d5975b70318b3","DTCompiler":"com.apple.compilers.llvm.clang.1_0"} diff --git a/tests/fixtures/generic/plist-safari-info.json b/tests/fixtures/generic/plist-safari-info.json new file mode 100644 index 000000000..d78d0e094 --- /dev/null +++ b/tests/fixtures/generic/plist-safari-info.json @@ -0,0 +1 @@ +{"ASWebAuthenticationSessionWebBrowserSupportCapabilities":{"EphemeralBrowserSessionIsSupported":true,"IsSupported":true},"Application-Group":["dot-mac","InternetAccounts"],"BuildMachineOSBuild":"20A241133","CFBundleDevelopmentRegion":"English","CFBundleDisplayName":"Safari","CFBundleDocumentTypes":[{"CFBundleTypeExtensions":["css"],"CFBundleTypeIconFile":"css.icns","CFBundleTypeIconSystemGenerated":"YES","CFBundleTypeMIMETypes":["text/css"],"CFBundleTypeName":"CSS style sheet","CFBundleTypeRole":"Viewer","LSHandlerRank":"Alternate","NSDocumentClass":"BrowserDocument"},{"CFBundleTypeExtensions":["pdf"],"CFBundleTypeIconFile":"pdf.icns","CFBundleTypeIconSystemGenerated":"YES","CFBundleTypeMIMETypes":["application/pdf"],"CFBundleTypeName":"PDF document","CFBundleTypeRole":"Viewer","LSHandlerRank":"Alternate","NSDocumentClass":"BrowserDocument"},{"CFBundleTypeExtensions":["webarchive"],"CFBundleTypeIconFile":"webarchive.icns","CFBundleTypeIconSystemGenerated":"YES","CFBundleTypeMIMETypes":["application/x-webarchive"],"CFBundleTypeName":"Web archive","CFBundleTypeRole":"Viewer","ICExtension":"ARCHIVE","LSHandlerRank":"Default","LSIsAppleDefaultForType":true,"NSDocumentClass":"BrowserDocument"},{"CFBundleTypeExtensions":["webbookmark"],"CFBundleTypeIconFile":"webbookmark.icns","CFBundleTypeIconSystemGenerated":"YES","CFBundleTypeName":"Safari bookmark","CFBundleTypeRole":"Viewer","LSHandlerRank":"Default","NSDocumentClass":"BrowserDocument"},{"CFBundleTypeExtensions":["webhistory"],"CFBundleTypeIconFile":"webhistory.icns","CFBundleTypeIconSystemGenerated":"YES","CFBundleTypeName":"Safari history item","CFBundleTypeRole":"Viewer","LSHandlerRank":"Default","NSDocumentClass":"BrowserDocument"},{"CFBundleTypeExtensions":["webloc"],"CFBundleTypeIconFile":"webloc.icns","CFBundleTypeIconSystemGenerated":"YES","CFBundleTypeName":"Web internet location","CFBundleTypeOSTypes":["ilht"],"CFBundleTypeRole":"Viewer","LSHandlerRank":"Default","NSDocumentClass":"BrowserDocument"},{"CFBundleTypeExtensions":["download"],"CFBundleTypeIconFile":"download10.icns","CFBundleTypeIconSystemGenerated":"YES","CFBundleTypeName":"Safari download","CFBundleTypeRole":"Editor","LSHandlerRank":"Default","LSTypeIsPackage":true,"NSDocumentClass":"BrowserDocument"},{"CFBundleTypeExtensions":["safariextz"],"CFBundleTypeIconFile":"safariextz.icns","CFBundleTypeIconSystemGenerated":"YES","CFBundleTypeMIMETypes":["application/x-safari-extension"],"CFBundleTypeName":"Safari extension","CFBundleTypeRole":"Viewer","LSHandlerRank":"Owner","LSTypeIsPackage":false,"NSDocumentClass":"BrowserDocument"},{"CFBundleTypeExtensions":["gif"],"CFBundleTypeIconFile":"gif.icns","CFBundleTypeIconSystemGenerated":"YES","CFBundleTypeMIMETypes":["image/gif"],"CFBundleTypeName":"GIF image","CFBundleTypeOSTypes":["GIFf"],"CFBundleTypeRole":"Viewer","LSHandlerRank":"Alternate","NSDocumentClass":"BrowserDocument"},{"CFBundleTypeExtensions":["html","htm","shtml","jhtml"],"CFBundleTypeIconFile":"html.icns","CFBundleTypeIconSystemGenerated":"YES","CFBundleTypeMIMETypes":["text/html"],"CFBundleTypeName":"HTML document","CFBundleTypeOSTypes":["HTML"],"CFBundleTypeRole":"Viewer","ICExtension":"HTML","LSHandlerRank":"Default","LSIsAppleDefaultForType":true,"NSDocumentClass":"BrowserDocument"},{"CFBundleTypeExtensions":["js"],"CFBundleTypeIconFile":"js.icns","CFBundleTypeIconSystemGenerated":"YES","CFBundleTypeMIMETypes":["application/x-javascript"],"CFBundleTypeName":"JavaScript script","CFBundleTypeRole":"Viewer","LSHandlerRank":"Alternate","NSDocumentClass":"BrowserDocument"},{"CFBundleTypeExtensions":["jpg","jpeg"],"CFBundleTypeIconFile":"jpeg.icns","CFBundleTypeIconSystemGenerated":"YES","CFBundleTypeMIMETypes":["image/jpeg"],"CFBundleTypeName":"JPEG image","CFBundleTypeOSTypes":["JPEG"],"CFBundleTypeRole":"Viewer","ICExtension":"JPEG","LSHandlerRank":"Alternate","NSDocumentClass":"BrowserDocument"},{"CFBundleTypeExtensions":["jp2"],"CFBundleTypeIconFile":"jp2.icns","CFBundleTypeIconSystemGenerated":"YES","CFBundleTypeMIMETypes":["image/jp2"],"CFBundleTypeName":"JPEG 2000 image","CFBundleTypeOSTypes":["jp2 "],"CFBundleTypeRole":"Viewer","LSHandlerRank":"Alternate","NSDocumentClass":"BrowserDocument"},{"CFBundleTypeExtensions":["txt","text"],"CFBundleTypeIconFile":"txt.icns","CFBundleTypeIconSystemGenerated":"YES","CFBundleTypeMIMETypes":["text/plain"],"CFBundleTypeName":"Plain text document","CFBundleTypeOSTypes":["TEXT"],"CFBundleTypeRole":"Viewer","ICExtension":"TXT","LSHandlerRank":"Alternate","NSDocumentClass":"BrowserDocument"},{"CFBundleTypeExtensions":["png"],"CFBundleTypeIconFile":"png.icns","CFBundleTypeMIMETypes":["image/png"],"CFBundleTypeName":"PNG image","CFBundleTypeOSTypes":["PNGf"],"CFBundleTypeRole":"Viewer","LSHandlerRank":"Alternate","NSDocumentClass":"BrowserDocument"},{"CFBundleTypeExtensions":["tiff","tif"],"CFBundleTypeIconFile":"tiff.icns","CFBundleTypeIconSystemGenerated":"YES","CFBundleTypeMIMETypes":["image/tiff"],"CFBundleTypeName":"TIFF image","CFBundleTypeOSTypes":["TIFF"],"CFBundleTypeRole":"Viewer","ICExtension":"TIFF","LSHandlerRank":"Alternate","NSDocumentClass":"BrowserDocument"},{"CFBundleTypeExtensions":["url"],"CFBundleTypeIconFile":"url.icns","CFBundleTypeIconSystemGenerated":"YES","CFBundleTypeName":"Web site location","CFBundleTypeOSTypes":["LINK"],"CFBundleTypeRole":"Viewer","LSHandlerRank":"Default","LSIsAppleDefaultForType":true,"NSDocumentClass":"BrowserDocument"},{"CFBundleTypeExtensions":["ico"],"CFBundleTypeIconFile":"ico.icns","CFBundleTypeIconSystemGenerated":"YES","CFBundleTypeMIMETypes":["image/x-icon"],"CFBundleTypeName":"Windows icon image","CFBundleTypeOSTypes":["ICO "],"CFBundleTypeRole":"Viewer","LSHandlerRank":"Alternate","NSDocumentClass":"BrowserDocument"},{"CFBundleTypeExtensions":["xhtml","xht","xhtm","xht"],"CFBundleTypeIconFile":"xhtml.icns","CFBundleTypeIconSystemGenerated":"YES","CFBundleTypeMIMETypes":["application/xhtml+xml"],"CFBundleTypeName":"XHTML document","CFBundleTypeRole":"Viewer","ICExtension":"XHTML","LSHandlerRank":"Default","NSDocumentClass":"BrowserDocument"},{"CFBundleTypeExtensions":["xml","xbl","xsl","xslt"],"CFBundleTypeIconFile":"xml.icns","CFBundleTypeIconSystemGenerated":"YES","CFBundleTypeMIMETypes":["application/xml","text/xml"],"CFBundleTypeName":"XML document","CFBundleTypeRole":"Viewer","ICExtension":"XML","LSHandlerRank":"Default","NSDocumentClass":"BrowserDocument"},{"CFBundleTypeExtensions":["svg"],"CFBundleTypeIconFile":"svg.icns","CFBundleTypeIconSystemGenerated":"YES","CFBundleTypeMIMETypes":["image/svg+xml"],"CFBundleTypeName":"SVG document","CFBundleTypeRole":"Viewer","LSHandlerRank":"Alternate","NSDocumentClass":"BrowserDocument"}],"CFBundleExecutable":"Safari","CFBundleGetInfoString":"15.6, Copyright © 2003-2022 Apple Inc.","CFBundleHelpBookFolder":"Safari.help","CFBundleHelpBookName":"com.apple.Safari.help","CFBundleIconFile":"AppIcon","CFBundleIconName":"AppIcon","CFBundleIdentifier":"com.apple.Safari","CFBundleInfoDictionaryVersion":"6.0","CFBundleName":"Safari","CFBundlePackageType":"APPL","CFBundleShortVersionString":"15.6","CFBundleSignature":"sfri","CFBundleSupportedPlatforms":["MacOSX"],"CFBundleURLTypes":[{"CFBundleURLName":"Web site URL","CFBundleURLSchemes":["http","https"],"LSHandlerRank":"Default","LSIsAppleDefaultForScheme":true},{"CFBundleURLName":"Local file URL","CFBundleURLSchemes":["file"],"LSHandlerRank":"Default"},{"CFBundleURLSchemes":["prefs"],"LSHandlerRank":"None"}],"CFBundleVersion":"17613.3.9.1.5","DTCompiler":"com.apple.compilers.llvm.clang.1_0","DTPlatformBuild":"21G66","DTPlatformName":"macosx","DTPlatformVersion":"12.5","DTSDKBuild":"21G66","DTSDKName":"macosx12.5.internal","DTXcode":"1330","DTXcodeBuild":"13E6049a","HPDHelpProjectIdentifier":"safari","LSApplicationCategoryType":"public.app-category.productivity","LSFileQuarantineEnabled":true,"LSMinimumSystemVersion":"12.5.0","NSAppleScriptEnabled":"Yes","NSExtensionSDK":{"com.apple.Safari.web-extension":{"NSExtension":{"NSExtensionAttributes":{"NSExtensionPointName":"com.apple.Safari.web-extension","NSExtensionPointVersion":"1.0"},"NSExtensionHostEntitlement":"com.apple.private.can-load-any-content-blocker","NSExtensionProtocol":"NSObject","PrincipalClass":"NSObject","Subsystems":["NSSharingService_Subsystem"]},"XPCService":{"JoinExistingSession":true,"RunLoopType":"NSRunLoop","ServiceType":"Application","_AdditionalSubServices":{"apple-extension-service":true}}}},"NSLocationUsageDescription":"Websites you visit may request your location.","NSMainNibFile":"MainMenu","NSPrincipalClass":"BrowserApplication","NSServices":[{"NSKeyEquivalent":{"default":"L"},"NSMenuItem":{"default":"Search With %WebSearchProvider@"},"NSMessage":"searchWithWebSearchProvider","NSPortName":"Safari","NSSendTypes":["public.utf8-plain-text"]},{"NSMenuItem":{"default":"Add to Reading List"},"NSMessage":"addToReadingList","NSPortName":"Safari","NSRequiredContext":[{"NSTextContent":"URL"},{"NSLinkSchemes":["http","https"]}],"NSSendTypes":["public.rtf","public.utf8-plain-text"]}],"NSSpeechRecognitionUsageDescription":"Websites you visit may access speech recognition.","NSSupportsAutomaticTermination":true,"NSSupportsSuddenTermination":false,"NSUserActivityTypes":["NSUserActivityTypeBrowsingWeb"],"OSAScriptingDefinition":"Safari.sdef","UTExportedTypeDeclarations":[{"UTTypeConformsTo":["public.data"],"UTTypeDescription":"Safari bookmark","UTTypeIdentifier":"com.apple.safari.bookmark","UTTypeTagSpecification":{"public.filename-extension":["webbookmark"]}},{"UTTypeConformsTo":["public.data"],"UTTypeDescription":"Safari extension","UTTypeIdentifier":"com.apple.safari.extension","UTTypeTagSpecification":{"public.filename-extension":["safariextz"]}},{"UTTypeConformsTo":["public.data"],"UTTypeDescription":"Safari history item","UTTypeIdentifier":"com.apple.safari.history","UTTypeTagSpecification":{"public.filename-extension":["webhistory"]}}]} diff --git a/tests/test_plist.py b/tests/test_plist.py new file mode 100644 index 000000000..432be1d01 --- /dev/null +++ b/tests/test_plist.py @@ -0,0 +1,75 @@ +import os +import unittest +import json +import jc.parsers.plist + +THIS_DIR = os.path.dirname(os.path.abspath(__file__)) + + +class MyTests(unittest.TestCase): + + def setUp(self): + # input + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/plist-garageband-info.plist'), 'rb') as f: + self.generic_garageband = f.read() + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/plist-safari-info.plist'), 'r', encoding='utf-8') as f: + self.generic_safari = f.read() + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/plist-alltypes.plist'), 'r', encoding='utf-8') as f: + self.generic_alltypes = f.read() + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/plist-alltypes-bin.plist'), 'rb') as f: + self.generic_alltypes_bin = f.read() + + # output + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/plist-garageband-info.json'), 'r', encoding='utf-8') as f: + self.generic_garageband_json = json.loads(f.read()) + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/plist-safari-info.json'), 'r', encoding='utf-8') as f: + self.generic_safari_json = json.loads(f.read()) + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/plist-alltypes.json'), 'r', encoding='utf-8') as f: + self.generic_alltypes_json = json.loads(f.read()) + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/plist-alltypes-bin.json'), 'r', encoding='utf-8') as f: + self.generic_alltypes_bin_json = json.loads(f.read()) + + + def test_plist_nodata(self): + """ + Test 'plist' with no data + """ + self.assertEqual(jc.parsers.plist.parse('', quiet=True), {}) + + + def test_plist_binary(self): + """ + Test binary plist file (garage band) + """ + self.assertEqual(jc.parsers.plist.parse(self.generic_garageband, quiet=True), self.generic_garageband_json) + + + def test_plist_xml(self): + """ + Test XML plist file (safari) + """ + self.assertEqual(jc.parsers.plist.parse(self.generic_safari, quiet=True), self.generic_safari_json) + + + def test_plist_xml_alltypes(self): + """ + Test XML plist file with all object types + """ + self.assertEqual(jc.parsers.plist.parse(self.generic_alltypes, quiet=True), self.generic_alltypes_json) + + + def test_plist_bin_alltypes(self): + """ + Test binary plist file with all object types + """ + self.assertEqual(jc.parsers.plist.parse(self.generic_alltypes_bin, quiet=True), self.generic_alltypes_bin_json) + + +if __name__ == '__main__': + unittest.main() From 498a2a314be14f03bf393019776efe07ecbe56b8 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Tue, 2 Aug 2022 08:50:30 -0700 Subject: [PATCH 040/116] doc formatting --- jc/parsers/x509_cert.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jc/parsers/x509_cert.py b/jc/parsers/x509_cert.py index ef7187d9d..790122d21 100644 --- a/jc/parsers/x509_cert.py +++ b/jc/parsers/x509_cert.py @@ -150,7 +150,7 @@ ] } - Signed Certificate Timestamp List + Signed Certificate Timestamp List: { "extn_id": "signed_certificate_timestamp_list", "critical": boolean, From c57d3416b79448a466964c987f73e57f197f88f9 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Wed, 3 Aug 2022 17:40:39 -0700 Subject: [PATCH 041/116] add timestamp option to cli --- jc/cli.py | 32 +++++++++++++++++++++++++++++++- jc/cli_data.py | 1 + 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/jc/cli.py b/jc/cli.py index 7264b50f3..2ce496e83 100644 --- a/jc/cli.py +++ b/jc/cli.py @@ -5,11 +5,11 @@ import io import sys import os +from datetime import datetime, timezone import textwrap import signal import shlex import subprocess -from typing import List, Dict from .lib import (__version__, parser_info, all_parser_info, parsers, _get_parser, _parser_is_streaming, standard_parser_mod_list, plugin_parser_mod_list, streaming_parser_mod_list) @@ -440,6 +440,29 @@ def combined_exit_code(program_exit=0, jc_exit=0): return exit_code +def add_timestamp_to(list_or_dict, runtime, magic_exit_code): + """This function mutates a list or dict in place""" + run_iso = runtime.isoformat() + run_timestamp = runtime.timestamp() + timestamp_obj = { + '_jc_meta': { + 'run_timestamp': run_timestamp, + 'run_iso': run_iso + } + } + + if isinstance(list_or_dict, dict): + list_or_dict.update(timestamp_obj) + + elif isinstance(list_or_dict, list): + for item in list_or_dict: + item.update(timestamp_obj) + + else: + utils.error_message(['Parser returned an unsupported object type.']) + sys.exit(combined_exit_code(magic_exit_code, JC_ERROR_EXIT)) + + def main(): # break on ctrl-c keyboard interrupt signal.signal(signal.SIGINT, ctrlc) @@ -484,6 +507,7 @@ def main(): quiet = 'q' in options ignore_exceptions = options.count('q') > 1 raw = 'r' in options + timestamp = 't' in options unbuffer = 'u' in options version_info = 'v' in options yaml_out = 'y' in options @@ -521,6 +545,9 @@ def main(): utils._safe_print(zsh_completion()) sys.exit(0) + if timestamp: + run_dt_utc = datetime.now(timezone.utc) + # if magic syntax used, try to run the command and error if it's not found, etc. magic_stdout, magic_stderr, magic_exit_code = None, None, 0 if run_command: @@ -622,6 +649,9 @@ def main(): raw=raw, quiet=quiet) + if timestamp: + add_timestamp_to(result, run_dt_utc, magic_exit_code) + safe_print_out(result, pretty=pretty, env_colors=jc_colors, diff --git a/jc/cli_data.py b/jc/cli_data.py index 2b0814f5f..12c6de85e 100644 --- a/jc/cli_data.py +++ b/jc/cli_data.py @@ -10,6 +10,7 @@ '--pretty': ['p', 'pretty print output'], '--quiet': ['q', 'suppress warnings (double to ignore streaming errors)'], '--raw': ['r', 'raw output'], + '--timestamp': ['t', 'add timestamp information to output'], '--unbuffer': ['u', 'unbuffer output'], '--version': ['v', 'version info'], '--yaml-out': ['y', 'YAML output'], From 5c9cd4e4cd47461ff70a4a73e843ace9f630aa1b Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 4 Aug 2022 07:44:25 -0700 Subject: [PATCH 042/116] timestamp working on standard and streaming parsers. updates existing _jc_meta if exists --- jc/cli.py | 27 +++++++++++++++------------ jc/cli_data.py | 2 +- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/jc/cli.py b/jc/cli.py index 2ce496e83..c6edd86e2 100644 --- a/jc/cli.py +++ b/jc/cli.py @@ -442,21 +442,21 @@ def combined_exit_code(program_exit=0, jc_exit=0): def add_timestamp_to(list_or_dict, runtime, magic_exit_code): """This function mutates a list or dict in place""" - run_iso = runtime.isoformat() run_timestamp = runtime.timestamp() - timestamp_obj = { - '_jc_meta': { - 'run_timestamp': run_timestamp, - 'run_iso': run_iso - } - } + timestamp_obj = {'timestamp': run_timestamp} if isinstance(list_or_dict, dict): - list_or_dict.update(timestamp_obj) + if '_jc_meta' in list_or_dict: + list_or_dict['_jc_meta'].update(timestamp_obj) + else: + list_or_dict['_jc_meta'] = timestamp_obj elif isinstance(list_or_dict, list): for item in list_or_dict: - item.update(timestamp_obj) + if '_jc_meta' in item: + item['_jc_meta'].update(timestamp_obj) + else: + item['_jc_meta'] = timestamp_obj else: utils.error_message(['Parser returned an unsupported object type.']) @@ -545,9 +545,6 @@ def main(): utils._safe_print(zsh_completion()) sys.exit(0) - if timestamp: - run_dt_utc = datetime.now(timezone.utc) - # if magic syntax used, try to run the command and error if it's not found, etc. magic_stdout, magic_stderr, magic_exit_code = None, None, 0 if run_command: @@ -623,7 +620,12 @@ def main(): raw=raw, quiet=quiet, ignore_exceptions=ignore_exceptions) + for line in result: + if timestamp: + run_dt_utc = datetime.now(timezone.utc) + add_timestamp_to(line, run_dt_utc, magic_exit_code) + safe_print_out(line, pretty=pretty, env_colors=jc_colors, @@ -650,6 +652,7 @@ def main(): quiet=quiet) if timestamp: + run_dt_utc = datetime.now(timezone.utc) add_timestamp_to(result, run_dt_utc, magic_exit_code) safe_print_out(result, diff --git a/jc/cli_data.py b/jc/cli_data.py index 12c6de85e..d07a22955 100644 --- a/jc/cli_data.py +++ b/jc/cli_data.py @@ -10,7 +10,7 @@ '--pretty': ['p', 'pretty print output'], '--quiet': ['q', 'suppress warnings (double to ignore streaming errors)'], '--raw': ['r', 'raw output'], - '--timestamp': ['t', 'add timestamp information to output'], + '--timestamp': ['t', 'add UTC Unix timestamp information to output'], '--unbuffer': ['u', 'unbuffer output'], '--version': ['v', 'version info'], '--yaml-out': ['y', 'YAML output'], From bfe41d03913e66a70edff1a953f394208d0a1e0b Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 4 Aug 2022 07:44:36 -0700 Subject: [PATCH 043/116] doc formatting --- jc/parsers/timestamp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jc/parsers/timestamp.py b/jc/parsers/timestamp.py index d271527f1..55891dd25 100644 --- a/jc/parsers/timestamp.py +++ b/jc/parsers/timestamp.py @@ -1,4 +1,4 @@ -"""jc - JSON Convert UNIX Epoch Timestamp string parser +"""jc - JSON Convert Unix Epoch Timestamp string parser The naive fields are based on the local time of the system the parser is run on. @@ -98,7 +98,7 @@ class info(): """Provides parser metadata (version, author, etc.)""" version = '1.0' - description = 'UNIX Epoch Timestamp string parser' + description = 'Unix Epoch Timestamp string parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' compatible = ['linux', 'aix', 'freebsd', 'darwin', 'win32', 'cygwin'] From 934ef046945d021b1ea2d92ca0b7dae79e9ab390 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 4 Aug 2022 08:27:48 -0700 Subject: [PATCH 044/116] change long timestamp option to --time-out so it doesn't clash with the timestamp parser name --- jc/cli_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jc/cli_data.py b/jc/cli_data.py index d07a22955..489f2126c 100644 --- a/jc/cli_data.py +++ b/jc/cli_data.py @@ -10,7 +10,7 @@ '--pretty': ['p', 'pretty print output'], '--quiet': ['q', 'suppress warnings (double to ignore streaming errors)'], '--raw': ['r', 'raw output'], - '--timestamp': ['t', 'add UTC Unix timestamp information to output'], + '--time-out': ['t', 'add UTC Unix timestamp information to output'], '--unbuffer': ['u', 'unbuffer output'], '--version': ['v', 'version info'], '--yaml-out': ['y', 'YAML output'], From ba39f35a01425dcf8fecf7a73ff3a0aa47394bc8 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 4 Aug 2022 11:28:22 -0700 Subject: [PATCH 045/116] disable YAML aliases --- jc/cli.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/jc/cli.py b/jc/cli.py index c6edd86e2..77eaaf872 100644 --- a/jc/cli.py +++ b/jc/cli.py @@ -253,17 +253,22 @@ def yaml_out(data, pretty=False, env_colors=None, mono=False, piped_out=False, a warning message to STDERR""" # make ruamel.yaml import optional try: - from ruamel.yaml import YAML + from ruamel.yaml import YAML, representer YAML_INSTALLED = True except Exception: YAML_INSTALLED = False if YAML_INSTALLED: y_string_buf = io.BytesIO() + # monkey patch to disable plugins since we don't use them and in # ruamel.yaml versions prior to 0.17.0 the use of __file__ in the # plugin code is incompatible with the pyoxidizer packager YAML.official_plug_ins = lambda a: [] + + # monkey patch to disable aliases + representer.RoundTripRepresenter.ignore_aliases = lambda x, y: True + yaml = YAML() yaml.default_flow_style = False yaml.explicit_start = True From 3166600ac69ddbb4ebabf5f9fd9c117e2e4809c7 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 4 Aug 2022 11:43:49 -0700 Subject: [PATCH 046/116] doc update --- CHANGELOG | 3 +++ README.md | 3 ++- completions/jc_bash_completion.sh | 2 +- completions/jc_zsh_completion.sh | 6 ++++-- docs/parsers/plist.md | 6 ++++++ docs/parsers/timestamp.md | 2 +- docs/parsers/x509_cert.md | 2 +- jc/parsers/plist.py | 2 ++ man/jc.1 | 8 ++++++-- templates/manpage_template | 4 ++++ templates/readme_template | 1 + 11 files changed, 31 insertions(+), 8 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 7192d0e75..40c7e7106 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -2,6 +2,9 @@ jc changelog xxxxxxxx v1.20.5 - Add IP Address string parser +- Add PLIST file parser (XML and binary support) +- Add `mdadm` command parser tested on linux (IN PROGRESS) +- Add `--time-out` or `-t` option to add a UTC timestamp to the JSON output - Fix `lsusb` command parser for output containing a `Device Qualifier` and `Binary Object Store Descriptor` sections - Change `LANG=C` to `LC_ALL=C` in locale instructions diff --git a/README.md b/README.md index 278f64207..94df9da5a 100644 --- a/README.md +++ b/README.md @@ -240,7 +240,7 @@ option. | ` --systeminfo` | `systeminfo` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/systeminfo) | | ` --time` | `/usr/bin/time` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/time) | | ` --timedatectl` | `timedatectl status` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/timedatectl) | -| ` --timestamp` | UNIX Epoch Timestamp string parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/timestamp) | +| ` --timestamp` | Unix Epoch Timestamp string parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/timestamp) | | ` --top` | `top -b` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/top) | | ` --top-s` | `top -b` command streaming parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/top_s) | | ` --tracepath` | `tracepath` and `tracepath6` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/tracepath) | @@ -276,6 +276,7 @@ option. | `-p` | `--pretty` | Pretty format the JSON output | | `-q` | `--quiet` | Quiet mode. Suppresses parser warning messages (use `-qq` to ignore streaming parser errors) | | `-r` | `--raw` | Raw output. Provides more literal output, typically with string values and no additional semantic processing | +| `-t` | `--time-out` | Add UTC Unix timestamp information to output | | `-u` | `--unbuffer` | Unbuffer output | | `-v` | `--version` | Version information | | `-y` | `--yaml-out` | YAML output | diff --git a/completions/jc_bash_completion.sh b/completions/jc_bash_completion.sh index 0e31b0008..b1230cc8a 100644 --- a/completions/jc_bash_completion.sh +++ b/completions/jc_bash_completion.sh @@ -5,7 +5,7 @@ _jc() jc_commands=(acpi airport arp blkid chage cksum crontab date df dig dmidecode dpkg du env file finger free git gpg hciconfig id ifconfig iostat iptables iw jobs last lastb ls lsblk lsmod lsof lsusb md5 md5sum mount mpstat netstat nmcli ntpq pidstat ping ping6 pip pip3 postconf printenv ps route rpm rsync sfdisk sha1sum sha224sum sha256sum sha384sum sha512sum shasum ss stat sum sysctl systemctl systeminfo timedatectl top tracepath tracepath6 traceroute traceroute6 ufw uname update-alternatives upower uptime vdir vmstat w wc who xrandr zipinfo) jc_parsers=(--acpi --airport --airport-s --arp --asciitable --asciitable-m --blkid --chage --cksum --crontab --crontab-u --csv --csv-s --date --df --dig --dir --dmidecode --dpkg-l --du --email-address --env --file --finger --free --fstab --git-log --git-log-s --gpg --group --gshadow --hash --hashsum --hciconfig --history --hosts --id --ifconfig --ini --iostat --iostat-s --ip-address --iptables --iso-datetime --iw-scan --jar-manifest --jobs --jwt --kv --last --ls --ls-s --lsblk --lsmod --lsof --lsusb --m3u --mount --mpstat --mpstat-s --netstat --nmcli --ntpq --passwd --pidstat --pidstat-s --ping --ping-s --pip-list --pip-show --plist --postconf --ps --route --rpm-qi --rsync --rsync-s --sfdisk --shadow --ss --stat --stat-s --sysctl --systemctl --systemctl-lj --systemctl-ls --systemctl-luf --systeminfo --time --timedatectl --timestamp --top --top-s --tracepath --traceroute --ufw --ufw-appinfo --uname --update-alt-gs --update-alt-q --upower --uptime --url --vmstat --vmstat-s --w --wc --who --x509-cert --xml --xrandr --yaml --zipinfo) - jc_options=(--force-color -C --debug -d --monochrome -m --pretty -p --quiet -q --raw -r --unbuffer -u --yaml-out -y) + jc_options=(--force-color -C --debug -d --monochrome -m --pretty -p --quiet -q --raw -r --time-out -t --unbuffer -u --yaml-out -y) jc_about_options=(--about -a) jc_about_mod_options=(--pretty -p --yaml-out -y --monochrome -m --force-color -C) jc_help_options=(--help -h) diff --git a/completions/jc_zsh_completion.sh b/completions/jc_zsh_completion.sh index 38614a4dd..153489dfe 100644 --- a/completions/jc_zsh_completion.sh +++ b/completions/jc_zsh_completion.sh @@ -186,7 +186,7 @@ _jc() { '--systeminfo:`systeminfo` command parser' '--time:`/usr/bin/time` command parser' '--timedatectl:`timedatectl status` command parser' - '--timestamp:UNIX Epoch Timestamp string parser' + '--timestamp:Unix Epoch Timestamp string parser' '--top:`top -b` command parser' '--top-s:`top -b` command streaming parser' '--tracepath:`tracepath` and `tracepath6` command parser' @@ -210,7 +210,7 @@ _jc() { '--yaml:YAML file parser' '--zipinfo:`zipinfo` command parser' ) - jc_options=(--force-color -C --debug -d --monochrome -m --pretty -p --quiet -q --raw -r --unbuffer -u --yaml-out -y) + jc_options=(--force-color -C --debug -d --monochrome -m --pretty -p --quiet -q --raw -r --time-out -t --unbuffer -u --yaml-out -y) jc_options_describe=( '--force-color:force color output even when using pipes (overrides -m)' '-C:force color output even when using pipes (overrides -m)' @@ -224,6 +224,8 @@ _jc() { '-q:suppress warnings (double to ignore streaming errors)' '--raw:raw output' '-r:raw output' + '--time-out:add UTC Unix timestamp information to output' + '-t:add UTC Unix timestamp information to output' '--unbuffer:unbuffer output' '-u:unbuffer output' '--yaml-out:YAML output' diff --git a/docs/parsers/plist.md b/docs/parsers/plist.md index 0e1869d57..3e77c28ce 100644 --- a/docs/parsers/plist.md +++ b/docs/parsers/plist.md @@ -7,6 +7,12 @@ jc - JSON Convert PLIST file parser Converts binary and XML PLIST files. +Binary values are converted into an ASCII hex representation. + +Datetime objects are converted into Unix epoch timestamps and ISO strings. +The timestamp and ISO string will maintain the same naive or timezone-aware +properties as the object in the original PLIST file. + Usage (cli): $ cat file.plist | jc --plist diff --git a/docs/parsers/timestamp.md b/docs/parsers/timestamp.md index 9706a1bd0..a5d07abd6 100644 --- a/docs/parsers/timestamp.md +++ b/docs/parsers/timestamp.md @@ -3,7 +3,7 @@ # jc.parsers.timestamp -jc - JSON Convert UNIX Epoch Timestamp string parser +jc - JSON Convert Unix Epoch Timestamp string parser The naive fields are based on the local time of the system the parser is run on. diff --git a/docs/parsers/x509_cert.md b/docs/parsers/x509_cert.md index ad4ee7b56..e5208d857 100644 --- a/docs/parsers/x509_cert.md +++ b/docs/parsers/x509_cert.md @@ -155,7 +155,7 @@ Schema: ] } - Signed Certificate Timestamp List + Signed Certificate Timestamp List: { "extn_id": "signed_certificate_timestamp_list", "critical": boolean, diff --git a/jc/parsers/plist.py b/jc/parsers/plist.py index bf8122154..0dbbab524 100644 --- a/jc/parsers/plist.py +++ b/jc/parsers/plist.py @@ -5,6 +5,8 @@ Binary values are converted into an ASCII hex representation. Datetime objects are converted into Unix epoch timestamps and ISO strings. +The timestamp and ISO string will maintain the same naive or timezone-aware +properties as the object in the original PLIST file. Usage (cli): diff --git a/man/jc.1 b/man/jc.1 index faaf14de8..aa71bcded 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -1,4 +1,4 @@ -.TH jc 1 2022-07-31 1.20.5 "JSON Convert" +.TH jc 1 2022-08-04 1.20.5 "JSON Convert" .SH NAME \fBjc\fP \- JSON Convert JSONifies the output of many CLI tools and file-types .SH SYNOPSIS @@ -470,7 +470,7 @@ PLIST file parser .TP .B \fB--timestamp\fP -UNIX Epoch Timestamp string parser +Unix Epoch Timestamp string parser .TP .B @@ -623,6 +623,10 @@ Quiet mode. Suppresses parser warning messages (use -qq to ignore streaming pars Raw output. Provides more literal output, typically with string values and no additional semantic processing .TP .B +\fB-t\fP, \fB--time-out\fP +Add UTC Unix timestamp information to output +.TP +.B \fB-u\fP, \fB--unbuffer\fP Unbuffer output (useful for slow streaming data with streaming parsers) .TP diff --git a/templates/manpage_template b/templates/manpage_template index 5bb644b8f..26a973070 100644 --- a/templates/manpage_template +++ b/templates/manpage_template @@ -63,6 +63,10 @@ Quiet mode. Suppresses parser warning messages (use -qq to ignore streaming pars Raw output. Provides more literal output, typically with string values and no additional semantic processing .TP .B +\fB-t\fP, \fB--time-out\fP +Add UTC Unix timestamp information to output +.TP +.B \fB-u\fP, \fB--unbuffer\fP Unbuffer output (useful for slow streaming data with streaming parsers) .TP diff --git a/templates/readme_template b/templates/readme_template index 27bb1e5f0..32c5a0aaa 100644 --- a/templates/readme_template +++ b/templates/readme_template @@ -164,6 +164,7 @@ option. | `-p` | `--pretty` | Pretty format the JSON output | | `-q` | `--quiet` | Quiet mode. Suppresses parser warning messages (use `-qq` to ignore streaming parser errors) | | `-r` | `--raw` | Raw output. Provides more literal output, typically with string values and no additional semantic processing | +| `-t` | `--time-out` | Add UTC Unix timestamp information to output | | `-u` | `--unbuffer` | Unbuffer output | | `-v` | `--version` | Version information | | `-y` | `--yaml-out` | YAML output | From cb3484e1ab564598bee10397b95d85e339c31eaf Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 4 Aug 2022 15:43:44 -0700 Subject: [PATCH 047/116] doc update --- EXAMPLES.md | 84 +++++++++++++++++++++++++++++++++++++++++++ docs/parsers/plist.md | 35 +++++++++++------- jc/parsers/plist.py | 37 +++++++++++-------- 3 files changed, 129 insertions(+), 27 deletions(-) diff --git a/EXAMPLES.md b/EXAMPLES.md index 2bf236ab5..b7e1a096a 100644 --- a/EXAMPLES.md +++ b/EXAMPLES.md @@ -1665,6 +1665,65 @@ $ iostat | jc --iostat -p # or: jc -p iostat } ] ``` +### IP Address strings +```bash +echo 192.168.2.10/24 | jc --ip-address -p +``` +```json +{ + "version": 4, + "max_prefix_length": 32, + "ip": "192.168.2.10", + "ip_compressed": "192.168.2.10", + "ip_exploded": "192.168.2.10", + "scope_id": null, + "ipv4_mapped": null, + "six_to_four": null, + "teredo_client": null, + "teredo_server": null, + "dns_ptr": "10.2.168.192.in-addr.arpa", + "network": "192.168.2.0", + "broadcast": "192.168.2.255", + "hostmask": "0.0.0.255", + "netmask": "255.255.255.0", + "cidr_netmask": 24, + "hosts": 254, + "first_host": "192.168.2.1", + "last_host": "192.168.2.254", + "is_multicast": false, + "is_private": true, + "is_global": false, + "is_link_local": false, + "is_loopback": false, + "is_reserved": false, + "is_unspecified": false, + "int": { + "ip": 3232236042, + "network": 3232236032, + "broadcast": 3232236287, + "first_host": 3232236033, + "last_host": 3232236286 + }, + "hex": { + "ip": "c0:a8:02:0a", + "network": "c0:a8:02:00", + "broadcast": "c0:a8:02:ff", + "hostmask": "00:00:00:ff", + "netmask": "ff:ff:ff:00", + "first_host": "c0:a8:02:01", + "last_host": "c0:a8:02:fe" + }, + "bin": { + "ip": "11000000101010000000001000001010", + "network": "11000000101010000000001000000000", + "broadcast": "11000000101010000000001011111111", + "hostmask": "00000000000000000000000011111111", + "netmask": "11111111111111111111111100000000", + "first_host": "11000000101010000000001000000001", + "last_host": "11000000101010000000001011111110" + } +} +``` ### iptables ```bash iptables --line-numbers -v -L -t nat | jc --iptables -p # or: jc -p iptables --line-numbers -v -L -t nat @@ -2832,6 +2891,31 @@ pip show wrapt wheel | jc --pip-show -p # or: jc -p pip show wrapt whe } ] ``` +### PLIST files +```bash +cat info.plist | jc --plist -p +``` +```json +{ + "NSAppleScriptEnabled": true, + "LSMultipleInstancesProhibited": true, + "CFBundleInfoDictionaryVersion": "6.0", + "DTPlatformVersion": "GM", + "CFBundleIconFile": "GarageBand.icns", + "CFBundleName": "GarageBand", + "DTSDKName": "macosx10.13internal", + "NSSupportsAutomaticGraphicsSwitching": true, + "RevisionDate": "2018-12-03_14:10:56", + "UTImportedTypeDeclarations": [ + { + "UTTypeConformsTo": [ + "public.data", + "public.content" + ] + } + ] +} +``` ### postconf -M ```bash postconf -M | jc --postconf -p # or jc -p postconf -M diff --git a/docs/parsers/plist.md b/docs/parsers/plist.md index 3e77c28ce..14b34c783 100644 --- a/docs/parsers/plist.md +++ b/docs/parsers/plist.md @@ -20,25 +20,34 @@ Usage (cli): Usage (module): import jc - result = jc.parse('plist', plist_command_output) + result = jc.parse('plist', plist_file_output) Schema: - [ - { - "plist": string, - "bar": boolean, - "baz": integer - } - ] + { + "": string/integer/float/boolean/object/array/null + } Examples: - $ plist | jc --plist -p - [] - - $ plist | jc --plist -p -r - [] + $ cat info.plist | jc --plist -p + { + "NSAppleScriptEnabled": true, + "LSMultipleInstancesProhibited": true, + "CFBundleInfoDictionaryVersion": "6.0", + "DTPlatformVersion": "GM", + "CFBundleIconFile": "GarageBand.icns", + "CFBundleName": "GarageBand", + "DTSDKName": "macosx10.13internal", + "NSSupportsAutomaticGraphicsSwitching": true, + "RevisionDate": "2018-12-03_14:10:56", + "UTImportedTypeDeclarations": [ + { + "UTTypeConformsTo": [ + "public.data", + "public.content" + ... + } diff --git a/jc/parsers/plist.py b/jc/parsers/plist.py index 0dbbab524..8cce38377 100644 --- a/jc/parsers/plist.py +++ b/jc/parsers/plist.py @@ -15,27 +15,36 @@ Usage (module): import jc - result = jc.parse('plist', plist_command_output) + result = jc.parse('plist', plist_file_output) Schema: - [ - { - "plist": string, - "bar": boolean, - "baz": integer - } - ] + { + "": string/integer/float/boolean/object/array/null + } Examples: - $ plist | jc --plist -p - [] - - $ plist | jc --plist -p -r - [] + $ cat info.plist | jc --plist -p + { + "NSAppleScriptEnabled": true, + "LSMultipleInstancesProhibited": true, + "CFBundleInfoDictionaryVersion": "6.0", + "DTPlatformVersion": "GM", + "CFBundleIconFile": "GarageBand.icns", + "CFBundleName": "GarageBand", + "DTSDKName": "macosx10.13internal", + "NSSupportsAutomaticGraphicsSwitching": true, + "RevisionDate": "2018-12-03_14:10:56", + "UTImportedTypeDeclarations": [ + { + "UTTypeConformsTo": [ + "public.data", + "public.content" + ... + } """ -from typing import List, Dict, Union +from typing import Dict, Union import plistlib import binascii from datetime import datetime From 6d66557b112f9543018d0c5de5e6b242728f7d50 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 4 Aug 2022 17:49:18 -0700 Subject: [PATCH 048/116] add_timestamp_to tests in cli.py --- tests/test_cli.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/tests/test_cli.py b/tests/test_cli.py index 07ba93d54..3368f346f 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,4 +1,5 @@ import unittest +from datetime import datetime, timezone import pygments from pygments.token import (Name, Number, String, Keyword) import jc.cli @@ -254,5 +255,41 @@ def test_cli_about_jc(self): self.assertGreaterEqual(jc.cli.about_jc()['parser_count'], 55) self.assertEqual(jc.cli.about_jc()['parser_count'], len(jc.cli.about_jc()['parsers'])) + def test_add_timestamp_to_simple_dict(self): + list_or_dict = {'a': 1, 'b': 2} + runtime = datetime(2022, 8, 5, 0, 37, 9, 273349, tzinfo=timezone.utc) + magic_exit_code = 0 + expected = {'a': 1, 'b': 2, '_jc_meta': {'timestamp': 1659659829.273349}} + jc.cli.add_timestamp_to(list_or_dict, runtime, magic_exit_code) + + self.assertEqual(list_or_dict, expected) + + def test_add_timestamp_to_simple_list(self): + list_or_dict = [{'a': 1, 'b': 2},{'a': 3, 'b': 4}] + runtime = datetime(2022, 8, 5, 0, 37, 9, 273349, tzinfo=timezone.utc) + magic_exit_code = 0 + expected = [{'a': 1, 'b': 2, '_jc_meta': {'timestamp': 1659659829.273349}}, {'a': 3, 'b': 4, '_jc_meta': {'timestamp': 1659659829.273349}}] + jc.cli.add_timestamp_to(list_or_dict, runtime, magic_exit_code) + + self.assertEqual(list_or_dict, expected) + + def test_add_timestamp_to_dict_existing_meta(self): + list_or_dict = {'a': 1, 'b': 2, '_jc_meta': {'foo': 'bar'}} + runtime = datetime(2022, 8, 5, 0, 37, 9, 273349, tzinfo=timezone.utc) + magic_exit_code = 0 + expected = {'a': 1, 'b': 2, '_jc_meta': {'foo': 'bar', 'timestamp': 1659659829.273349}} + jc.cli.add_timestamp_to(list_or_dict, runtime, magic_exit_code) + + self.assertEqual(list_or_dict, expected) + + def test_add_timestamp_to_list_existing_meta(self): + list_or_dict = [{'a': 1, 'b': 2, '_jc_meta': {'foo': 'bar'}},{'a': 3, 'b': 4, '_jc_meta': {'foo': 'bar'}}] + runtime = datetime(2022, 8, 5, 0, 37, 9, 273349, tzinfo=timezone.utc) + magic_exit_code = 0 + expected = [{'a': 1, 'b': 2, '_jc_meta': {'foo': 'bar', 'timestamp': 1659659829.273349}}, {'a': 3, 'b': 4, '_jc_meta': {'foo': 'bar', 'timestamp': 1659659829.273349}}] + jc.cli.add_timestamp_to(list_or_dict, runtime, magic_exit_code) + + self.assertEqual(list_or_dict, expected) + if __name__ == '__main__': unittest.main() \ No newline at end of file From 123de1a3ce14011ea843fc888ea6a2c2cbc5a536 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 5 Aug 2022 10:01:00 -0700 Subject: [PATCH 049/116] simplify timestamp field addition --- jc/cli.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/jc/cli.py b/jc/cli.py index 77eaaf872..cffd4c14a 100644 --- a/jc/cli.py +++ b/jc/cli.py @@ -446,22 +446,27 @@ def combined_exit_code(program_exit=0, jc_exit=0): def add_timestamp_to(list_or_dict, runtime, magic_exit_code): - """This function mutates a list or dict in place""" + """ + This function mutates a list or dict in place. If the _jc_meta field + does not already exist, it will be created with the timestamp field. If + the _jc_meta field already exists, the timestamp will be added to the + existing object. + """ run_timestamp = runtime.timestamp() timestamp_obj = {'timestamp': run_timestamp} if isinstance(list_or_dict, dict): - if '_jc_meta' in list_or_dict: - list_or_dict['_jc_meta'].update(timestamp_obj) - else: - list_or_dict['_jc_meta'] = timestamp_obj + if '_jc_meta' not in list_or_dict: + list_or_dict['_jc_meta'] = {} + + list_or_dict['_jc_meta'].update(timestamp_obj) elif isinstance(list_or_dict, list): for item in list_or_dict: - if '_jc_meta' in item: - item['_jc_meta'].update(timestamp_obj) - else: - item['_jc_meta'] = timestamp_obj + if '_jc_meta' not in item: + item['_jc_meta'] = {} + + item['_jc_meta'].update(timestamp_obj) else: utils.error_message(['Parser returned an unsupported object type.']) From c15f7641f451cc6eb5ed20adf94a10d5180f50d4 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sat, 6 Aug 2022 17:05:03 -0700 Subject: [PATCH 050/116] doc update --- CHANGELOG | 1 + docs/parsers/timestamp.md | 4 ++-- jc/parsers/timestamp.py | 4 ++-- man/jc.1 | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 40c7e7106..a7d7cf562 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -9,6 +9,7 @@ xxxxxxxx v1.20.5 `Binary Object Store Descriptor` sections - Change `LANG=C` to `LC_ALL=C` in locale instructions - Add `__main__.py` to package allowing `python -m jc` usage +- Add an enclosing top-level folder inside the windows.zip package 20220723 v1.20.4 - Fix URL string parser path list for URLs ending in a forward slash diff --git a/docs/parsers/timestamp.md b/docs/parsers/timestamp.md index a5d07abd6..444a1cea5 100644 --- a/docs/parsers/timestamp.md +++ b/docs/parsers/timestamp.md @@ -12,7 +12,7 @@ The utc fields are timezone-aware, based on the UTC timezone. Usage (cli): - $ echo "2022-07-20T14:52:45Z" | jc --timestamp + $ echo 1658599410 | jc --timestamp Usage (module): @@ -59,7 +59,7 @@ Schema: Examples: - $ echo '1658599410' | jc --timestamp -p + $ echo 1658599410 | jc --timestamp -p { "naive": { "year": 2022, diff --git a/jc/parsers/timestamp.py b/jc/parsers/timestamp.py index 55891dd25..faf1226fb 100644 --- a/jc/parsers/timestamp.py +++ b/jc/parsers/timestamp.py @@ -7,7 +7,7 @@ Usage (cli): - $ echo "2022-07-20T14:52:45Z" | jc --timestamp + $ echo 1658599410 | jc --timestamp Usage (module): @@ -54,7 +54,7 @@ Examples: - $ echo '1658599410' | jc --timestamp -p + $ echo 1658599410 | jc --timestamp -p { "naive": { "year": 2022, diff --git a/man/jc.1 b/man/jc.1 index aa71bcded..b237c3d8e 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -1,4 +1,4 @@ -.TH jc 1 2022-08-04 1.20.5 "JSON Convert" +.TH jc 1 2022-08-06 1.20.5 "JSON Convert" .SH NAME \fBjc\fP \- JSON Convert JSONifies the output of many CLI tools and file-types .SH SYNOPSIS From bc1521d9374e61cc20530c8e9d55155ffec8d1e8 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sun, 7 Aug 2022 13:39:58 -0700 Subject: [PATCH 051/116] add initial mdadm parser --- jc/lib.py | 1 + jc/parsers/mdadm.py | 222 ++++++++++++++++++ .../generic/mdadm/examine-raid0-offline | 26 ++ tests/fixtures/generic/mdadm/examine-raid0-ok | 26 ++ .../generic/mdadm/examine-raid1-0.90-ok | 27 +++ .../generic/mdadm/examine-raid1-checking | 28 +++ .../generic/mdadm/examine-raid1-failfast | 27 +++ .../generic/mdadm/examine-raid1-faulty1 | 26 ++ .../generic/mdadm/examine-raid1-faulty2 | 26 ++ .../generic/mdadm/examine-raid1-moreflags | 27 +++ tests/fixtures/generic/mdadm/examine-raid1-ok | 26 ++ .../generic/mdadm/examine-raid1-replacing | 28 +++ .../generic/mdadm/examine-raid1-resync | 28 +++ .../generic/mdadm/examine-raid1-spare | 26 ++ .../generic/mdadm/examine-raid1-syncing | 26 ++ .../fixtures/generic/mdadm/mdadm-examine.out | 27 +++ .../generic/mdadm/mdadm-query-detail.out | 28 +++ tests/fixtures/generic/mdadm/query-raid0-ok | 28 +++ .../mdadm/query-raid1-failed-and-flags | 28 +++ .../fixtures/generic/mdadm/query-raid1-faulty | 28 +++ .../mdadm/query-raid1-faulty-with-spare | 30 +++ .../mdadm/query-raid1-faulty_and_removed | 26 ++ tests/fixtures/generic/mdadm/query-raid1-ok | 26 ++ .../fixtures/generic/mdadm/query-raid1-ok-0.9 | 26 ++ .../generic/mdadm/query-raid1-ok-failfast | 26 ++ .../generic/mdadm/query-raid1-ok-spare | 28 +++ .../mdadm/query-raid1-rebuild-failfast | 28 +++ .../mdadm/query-raid1-spare-writem-rebuild | 29 +++ .../generic/mdadm/query-raid1-syncing | 28 +++ 29 files changed, 956 insertions(+) create mode 100644 jc/parsers/mdadm.py create mode 100644 tests/fixtures/generic/mdadm/examine-raid0-offline create mode 100644 tests/fixtures/generic/mdadm/examine-raid0-ok create mode 100644 tests/fixtures/generic/mdadm/examine-raid1-0.90-ok create mode 100644 tests/fixtures/generic/mdadm/examine-raid1-checking create mode 100644 tests/fixtures/generic/mdadm/examine-raid1-failfast create mode 100644 tests/fixtures/generic/mdadm/examine-raid1-faulty1 create mode 100644 tests/fixtures/generic/mdadm/examine-raid1-faulty2 create mode 100644 tests/fixtures/generic/mdadm/examine-raid1-moreflags create mode 100644 tests/fixtures/generic/mdadm/examine-raid1-ok create mode 100644 tests/fixtures/generic/mdadm/examine-raid1-replacing create mode 100644 tests/fixtures/generic/mdadm/examine-raid1-resync create mode 100644 tests/fixtures/generic/mdadm/examine-raid1-spare create mode 100644 tests/fixtures/generic/mdadm/examine-raid1-syncing create mode 100644 tests/fixtures/generic/mdadm/mdadm-examine.out create mode 100644 tests/fixtures/generic/mdadm/mdadm-query-detail.out create mode 100644 tests/fixtures/generic/mdadm/query-raid0-ok create mode 100644 tests/fixtures/generic/mdadm/query-raid1-failed-and-flags create mode 100644 tests/fixtures/generic/mdadm/query-raid1-faulty create mode 100644 tests/fixtures/generic/mdadm/query-raid1-faulty-with-spare create mode 100644 tests/fixtures/generic/mdadm/query-raid1-faulty_and_removed create mode 100644 tests/fixtures/generic/mdadm/query-raid1-ok create mode 100644 tests/fixtures/generic/mdadm/query-raid1-ok-0.9 create mode 100644 tests/fixtures/generic/mdadm/query-raid1-ok-failfast create mode 100644 tests/fixtures/generic/mdadm/query-raid1-ok-spare create mode 100644 tests/fixtures/generic/mdadm/query-raid1-rebuild-failfast create mode 100644 tests/fixtures/generic/mdadm/query-raid1-spare-writem-rebuild create mode 100644 tests/fixtures/generic/mdadm/query-raid1-syncing diff --git a/jc/lib.py b/jc/lib.py index 4feff98f8..40314c829 100644 --- a/jc/lib.py +++ b/jc/lib.py @@ -66,6 +66,7 @@ 'lsof', 'lsusb', 'm3u', + 'mdadm', 'mount', 'mpstat', 'mpstat-s', diff --git a/jc/parsers/mdadm.py b/jc/parsers/mdadm.py new file mode 100644 index 000000000..874cbae6c --- /dev/null +++ b/jc/parsers/mdadm.py @@ -0,0 +1,222 @@ +"""jc - JSON Convert `mdadm` command output parser + +<> + +Usage (cli): + + $ mdadm | jc --mdadm + + or + + $ jc mdadm + +Usage (module): + + import jc + result = jc.parse('mdadm', mdadm_command_output) + +Schema: + + [ + { + "mdadm": string, + "bar": boolean, + "baz": integer + } + ] + +Examples: + + $ mdadm | jc --mdadm -p + [] + + $ mdadm | jc --mdadm -p -r + [] +""" +from typing import List, Dict +import jc.utils +from jc.parsers.universal import sparse_table_parse + + +class info(): + """Provides parser metadata (version, author, etc.)""" + version = '1.0' + description = '`mdadm` command parser' + author = 'Kelly Brazil' + author_email = 'kellyjonbrazil@gmail.com' + compatible = ['linux'] + magic_commands = ['mdadm'] + + +__version__ = info.version + + +def _process(proc_data: Dict) -> Dict: + """ + Final processing to conform to the schema. + + Parameters: + + proc_data: (List of Dictionaries) raw structured data to process + + Returns: + + List of Dictionaries. Structured to conform to the schema. + """ + int_list = {'array_size_num', 'used_dev_size_num', 'raid_devices', 'total_devices', + 'active_devices', 'working_devices', 'failed_devices', 'spare_devices', + 'events', 'number', 'major', 'minor', 'raid_device', 'avail_dev_size_num', + 'data_offset', 'super_offset', 'unused_space_before', 'unused_space_after', + 'chunk_size', 'preferred_minor', 'check_status_percent', 'resync_status_percent'} + + array_state_map = { + 'A': 'active', + '.': 'missing', + 'R': 'replacing' + } + + # split combined values + if 'array_size' in proc_data: + proc_data['array_size_num'] = proc_data['array_size'].split(maxsplit=1)[0] + + if 'used_dev_size' in proc_data: + proc_data['used_dev_size_num'] = proc_data['used_dev_size'].split(maxsplit=1)[0] + + if 'avail_dev_size' in proc_data: + proc_data['avail_dev_size_num'] = proc_data['avail_dev_size'].split(maxsplit=1)[0] + + if 'data_offset' in proc_data: + proc_data['data_offset'] = proc_data['data_offset'].split()[0] + + if 'super_offset' in proc_data: + proc_data['super_offset'] = proc_data['super_offset'].split()[0] + + if 'unused_space' in proc_data: + unused_space_list = proc_data['unused_space'].split(',') + before_text = unused_space_list[0].strip() + after_text = unused_space_list[1].strip() + before = before_text.split('=')[1].split()[0] + after = after_text.split('=')[1].split()[0] + proc_data['unused_space_before'] = before + proc_data['unused_space_after'] = after + + if 'name' in proc_data: + proc_data['name_val'] = proc_data['name'].split(maxsplit=1)[0] + + if 'checksum' in proc_data: + proc_data['checksum_val'] = proc_data['checksum'].split(maxsplit=1)[0] + proc_data['checksum_state'] = proc_data['checksum'].split()[-1] + + if 'state' in proc_data: + state_list = [x.strip() for x in proc_data['state'].split(',')] + proc_data['state_list'] = state_list + + if 'flags' in proc_data: + proc_data['flag_list'] = proc_data['flags'].split() + + if 'array_state' in proc_data: + array_state_list = [] + array_state_text = proc_data['array_state'].split(maxsplit=1)[0] + + for item in array_state_text: + array_state_list.append(array_state_map[item]) + + proc_data['array_state_list'] = array_state_list + + if 'resync_status' in proc_data: + proc_data['resync_status_percent'] = proc_data['resync_status'].split('%')[0] + + if 'check_status' in proc_data: + proc_data['check_status_percent'] = proc_data['check_status'].split('%')[0] + + # add timestamp fields + if 'creation_time' in proc_data: + dt = jc.utils.timestamp(proc_data['creation_time'], format_hint=(1000,)) + proc_data['creation_time_epoch'] = dt.naive + + if 'update_time' in proc_data: + dt = jc.utils.timestamp(proc_data['update_time'], format_hint=(1000,)) + proc_data['update_time_epoch'] = dt.naive + + # convert ints + for key in proc_data: + if key in int_list: + proc_data[key] = jc .utils.convert_to_int(proc_data[key]) + + # table items + if 'device_table' in proc_data: + for item in proc_data['device_table']: + if 'state' in item: + item['state'] = item['state'].split() + + # convert ints + for key in item: + if key in int_list: + item[key] = jc.utils.convert_to_int(item[key]) + + return proc_data + + +def parse( + data: str, + raw: bool = False, + quiet: bool = False +) -> Dict: + """ + Main text parsing function + + Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + + Returns: + + List of Dictionaries. Raw or processed structured data. + """ + jc.utils.compatibility(__name__, info.compatible, quiet) + jc.utils.input_type_check(data) + + raw_output = {} + device_table = False + device_table_list = [] + + if jc.utils.has_data(data): + + for line in filter(None, data.splitlines()): + # device line + if line.startswith('/') and line.endswith(':'): + raw_output['device'] = line[:-1] + continue + + # key/value lines + if ' : ' in line: + key, value = line.split(' : ') + key = key.strip().lower().replace(' ', '_') + value = value.strip() + raw_output[key] = value + continue + + # device table header + if ' Number Major Minor RaidDevice State' in line: + device_table_list.append(' number major minor RaidDevice state device') + device_table = True + continue + + # device table lines + if device_table == True: + device_table_list.append(line) + continue + + if device_table_list: + d_table = sparse_table_parse(device_table_list) + for item in d_table: + if 'RaidDevice' in item: + item['raid_device'] = item.pop('RaidDevice') + + raw_output['device_table'] = d_table + + + + return raw_output if raw else _process(raw_output) diff --git a/tests/fixtures/generic/mdadm/examine-raid0-offline b/tests/fixtures/generic/mdadm/examine-raid0-offline new file mode 100644 index 000000000..255ef29f1 --- /dev/null +++ b/tests/fixtures/generic/mdadm/examine-raid0-offline @@ -0,0 +1,26 @@ +/dev/vda1: + Magic : a92b4efc + Version : 1.2 + Feature Map : 0x0 + Array UUID : 7e81a856:abb9c1c2:4b71237a:9778cc66 + Name : sysrescue:0 (local to host sysrescue) + Creation Time : Sun Aug 7 20:56:52 2022 + Raid Level : raid0 + Raid Devices : 2 + + Avail Dev Size : 200704 sectors (98.00 MiB 102.76 MB) + Data Offset : 4096 sectors + Super Offset : 8 sectors + Unused Space : before=4016 sectors, after=0 sectors + State : clean + Device UUID : a49d96f1:8e86ca03:5ff499ec:8cdd42bb + + Update Time : Sun Aug 7 20:56:52 2022 + Bad Block Log : 512 entries available at offset 8 sectors + Checksum : eeb6cbe3 - correct + Events : 0 + + Chunk Size : 512K + + Device Role : Active device 0 + Array State : AA ('A' == active, '.' == missing, 'R' == replacing) diff --git a/tests/fixtures/generic/mdadm/examine-raid0-ok b/tests/fixtures/generic/mdadm/examine-raid0-ok new file mode 100644 index 000000000..255ef29f1 --- /dev/null +++ b/tests/fixtures/generic/mdadm/examine-raid0-ok @@ -0,0 +1,26 @@ +/dev/vda1: + Magic : a92b4efc + Version : 1.2 + Feature Map : 0x0 + Array UUID : 7e81a856:abb9c1c2:4b71237a:9778cc66 + Name : sysrescue:0 (local to host sysrescue) + Creation Time : Sun Aug 7 20:56:52 2022 + Raid Level : raid0 + Raid Devices : 2 + + Avail Dev Size : 200704 sectors (98.00 MiB 102.76 MB) + Data Offset : 4096 sectors + Super Offset : 8 sectors + Unused Space : before=4016 sectors, after=0 sectors + State : clean + Device UUID : a49d96f1:8e86ca03:5ff499ec:8cdd42bb + + Update Time : Sun Aug 7 20:56:52 2022 + Bad Block Log : 512 entries available at offset 8 sectors + Checksum : eeb6cbe3 - correct + Events : 0 + + Chunk Size : 512K + + Device Role : Active device 0 + Array State : AA ('A' == active, '.' == missing, 'R' == replacing) diff --git a/tests/fixtures/generic/mdadm/examine-raid1-0.90-ok b/tests/fixtures/generic/mdadm/examine-raid1-0.90-ok new file mode 100644 index 000000000..97be20ee6 --- /dev/null +++ b/tests/fixtures/generic/mdadm/examine-raid1-0.90-ok @@ -0,0 +1,27 @@ +/dev/vda1: + Magic : a92b4efc + Version : 0.90.00 + UUID : 3f720601:29af5283:96fc04a8:108a4af7 (local to host sysrescue) + Creation Time : Sun Aug 7 21:52:56 2022 + Raid Level : raid1 + Used Dev Size : 102336 (99.94 MiB 104.79 MB) + Array Size : 102336 (99.94 MiB 104.79 MB) + Raid Devices : 2 + Total Devices : 2 +Preferred Minor : 0 + + Update Time : Sun Aug 7 21:54:35 2022 + State : clean + Active Devices : 2 +Working Devices : 2 + Failed Devices : 0 + Spare Devices : 0 + Checksum : 7fb4bb0e - correct + Events : 18 + + + Number Major Minor RaidDevice State +this 0 254 1 0 active sync /dev/vda1 + + 0 0 254 1 0 active sync /dev/vda1 + 1 1 254 2 1 active sync /dev/vda2 diff --git a/tests/fixtures/generic/mdadm/examine-raid1-checking b/tests/fixtures/generic/mdadm/examine-raid1-checking new file mode 100644 index 000000000..161537515 --- /dev/null +++ b/tests/fixtures/generic/mdadm/examine-raid1-checking @@ -0,0 +1,28 @@ +/dev/md126: + Version : 1.2 + Creation Time : Sun Aug 7 21:15:49 2022 + Raid Level : raid1 + Array Size : 101376 (99.00 MiB 103.81 MB) + Used Dev Size : 101376 (99.00 MiB 103.81 MB) + Raid Devices : 2 + Total Devices : 2 + Persistence : Superblock is persistent + + Update Time : Sun Aug 7 21:23:59 2022 + State : clean, checking + Active Devices : 2 + Working Devices : 2 + Failed Devices : 0 + Spare Devices : 0 + +Consistency Policy : resync + + Check Status : 21% complete + + Name : sysrescue:126 (local to host sysrescue) + UUID : e054553a:fbccdcfe:0ae80bc8:9379377a + Events : 18 + + Number Major Minor RaidDevice State + 0 254 1 0 active sync /dev/vda1 + 1 254 2 1 active sync /dev/vda2 diff --git a/tests/fixtures/generic/mdadm/examine-raid1-failfast b/tests/fixtures/generic/mdadm/examine-raid1-failfast new file mode 100644 index 000000000..8fcd9d5cc --- /dev/null +++ b/tests/fixtures/generic/mdadm/examine-raid1-failfast @@ -0,0 +1,27 @@ +/dev/vda2: + Magic : a92b4efc + Version : 1.2 + Feature Map : 0x0 + Array UUID : e054553a:fbccdcfe:0ae80bc8:9379377a + Name : sysrescue:126 (local to host sysrescue) + Creation Time : Sun Aug 7 21:15:49 2022 + Raid Level : raid1 + Raid Devices : 2 + + Avail Dev Size : 202752 sectors (99.00 MiB 103.81 MB) + Array Size : 101376 KiB (99.00 MiB 103.81 MB) + Data Offset : 2048 sectors + Super Offset : 8 sectors + Unused Space : before=1968 sectors, after=0 sectors + State : clean + Device UUID : 140a340b:69d5759c:b87ce67c:6579ce22 + + Flags : failfast + Update Time : Sun Aug 7 21:37:34 2022 + Bad Block Log : 512 entries available at offset 16 sectors + Checksum : 4bf42c40 - correct + Events : 51 + + + Device Role : Active device 1 + Array State : AA ('A' == active, '.' == missing, 'R' == replacing) diff --git a/tests/fixtures/generic/mdadm/examine-raid1-faulty1 b/tests/fixtures/generic/mdadm/examine-raid1-faulty1 new file mode 100644 index 000000000..860b91ff8 --- /dev/null +++ b/tests/fixtures/generic/mdadm/examine-raid1-faulty1 @@ -0,0 +1,26 @@ +/dev/vda1: + Magic : a92b4efc + Version : 1.2 + Feature Map : 0x0 + Array UUID : e054553a:fbccdcfe:0ae80bc8:9379377a + Name : sysrescue:126 (local to host sysrescue) + Creation Time : Sun Aug 7 21:15:49 2022 + Raid Level : raid1 + Raid Devices : 2 + + Avail Dev Size : 202752 sectors (99.00 MiB 103.81 MB) + Array Size : 101376 KiB (99.00 MiB 103.81 MB) + Data Offset : 2048 sectors + Super Offset : 8 sectors + Unused Space : before=1968 sectors, after=0 sectors + State : clean + Device UUID : 1ec6a01d:e96d7f47:59970cf2:f0c056d2 + + Update Time : Sun Aug 7 21:28:34 2022 + Bad Block Log : 512 entries available at offset 16 sectors + Checksum : 2e18e0bb - correct + Events : 25 + + + Device Role : Active device 0 + Array State : A. ('A' == active, '.' == missing, 'R' == replacing) diff --git a/tests/fixtures/generic/mdadm/examine-raid1-faulty2 b/tests/fixtures/generic/mdadm/examine-raid1-faulty2 new file mode 100644 index 000000000..48f218aef --- /dev/null +++ b/tests/fixtures/generic/mdadm/examine-raid1-faulty2 @@ -0,0 +1,26 @@ +/dev/vda2: + Magic : a92b4efc + Version : 1.2 + Feature Map : 0x0 + Array UUID : e054553a:fbccdcfe:0ae80bc8:9379377a + Name : sysrescue:126 (local to host sysrescue) + Creation Time : Sun Aug 7 21:15:49 2022 + Raid Level : raid1 + Raid Devices : 2 + + Avail Dev Size : 202752 sectors (99.00 MiB 103.81 MB) + Array Size : 101376 KiB (99.00 MiB 103.81 MB) + Data Offset : 2048 sectors + Super Offset : 8 sectors + Unused Space : before=1968 sectors, after=0 sectors + State : clean + Device UUID : 66e4da05:9cc62990:0620d976:579c0948 + + Update Time : Sun Aug 7 21:27:02 2022 + Bad Block Log : 512 entries available at offset 16 sectors + Checksum : 597fbb6b - correct + Events : 23 + + + Device Role : Active device 1 + Array State : AA ('A' == active, '.' == missing, 'R' == replacing) diff --git a/tests/fixtures/generic/mdadm/examine-raid1-moreflags b/tests/fixtures/generic/mdadm/examine-raid1-moreflags new file mode 100644 index 000000000..34455fa3f --- /dev/null +++ b/tests/fixtures/generic/mdadm/examine-raid1-moreflags @@ -0,0 +1,27 @@ +/dev/vda2: + Magic : a92b4efc + Version : 1.2 + Feature Map : 0x0 + Array UUID : e054553a:fbccdcfe:0ae80bc8:9379377a + Name : sysrescue:126 (local to host sysrescue) + Creation Time : Sun Aug 7 21:15:49 2022 + Raid Level : raid1 + Raid Devices : 2 + + Avail Dev Size : 202752 sectors (99.00 MiB 103.81 MB) + Array Size : 101376 KiB (99.00 MiB 103.81 MB) + Data Offset : 2048 sectors + Super Offset : 8 sectors + Unused Space : before=1968 sectors, after=0 sectors + State : clean + Device UUID : cc336e3b:67a49dea:480135ed:9e1280cd + + Flags : write-mostly failfast + Update Time : Sun Aug 7 21:48:58 2022 + Bad Block Log : 512 entries available at offset 16 sectors + Checksum : e5554588 - correct + Events : 78 + + + Device Role : Active device 1 + Array State : AA ('A' == active, '.' == missing, 'R' == replacing) diff --git a/tests/fixtures/generic/mdadm/examine-raid1-ok b/tests/fixtures/generic/mdadm/examine-raid1-ok new file mode 100644 index 000000000..192e1efc6 --- /dev/null +++ b/tests/fixtures/generic/mdadm/examine-raid1-ok @@ -0,0 +1,26 @@ +/dev/vda1: + Magic : a92b4efc + Version : 1.2 + Feature Map : 0x0 + Array UUID : e054553a:fbccdcfe:0ae80bc8:9379377a + Name : sysrescue:126 (local to host sysrescue) + Creation Time : Sun Aug 7 21:15:49 2022 + Raid Level : raid1 + Raid Devices : 2 + + Avail Dev Size : 202752 sectors (99.00 MiB 103.81 MB) + Array Size : 101376 KiB (99.00 MiB 103.81 MB) + Data Offset : 2048 sectors + Super Offset : 8 sectors + Unused Space : before=1968 sectors, after=0 sectors + State : clean + Device UUID : 1ec6a01d:e96d7f47:59970cf2:f0c056d2 + + Update Time : Sun Aug 7 21:20:08 2022 + Bad Block Log : 512 entries available at offset 16 sectors + Checksum : 2e1bdeb8 - correct + Events : 17 + + + Device Role : Active device 0 + Array State : AA ('A' == active, '.' == missing, 'R' == replacing) diff --git a/tests/fixtures/generic/mdadm/examine-raid1-replacing b/tests/fixtures/generic/mdadm/examine-raid1-replacing new file mode 100644 index 000000000..dfb99ec42 --- /dev/null +++ b/tests/fixtures/generic/mdadm/examine-raid1-replacing @@ -0,0 +1,28 @@ +/dev/vda2: + Magic : a92b4efc + Version : 1.2 + Feature Map : 0x12 + Array UUID : e054553a:fbccdcfe:0ae80bc8:9379377a + Name : sysrescue:126 (local to host sysrescue) + Creation Time : Sun Aug 7 21:15:49 2022 + Raid Level : raid1 + Raid Devices : 2 + + Avail Dev Size : 202752 sectors (99.00 MiB 103.81 MB) + Array Size : 101376 KiB (99.00 MiB 103.81 MB) + Data Offset : 2048 sectors + Super Offset : 8 sectors +Recovery Offset : 115200 sectors + Unused Space : before=1968 sectors, after=0 sectors + State : clean + Device UUID : cc336e3b:67a49dea:480135ed:9e1280cd + + Flags : write-mostly failfast + Update Time : Sun Aug 7 21:48:15 2022 + Bad Block Log : 512 entries available at offset 16 sectors + Checksum : e55a076b - correct + Events : 75 + + + Device Role : Replacement device 1 + Array State : AR ('A' == active, '.' == missing, 'R' == replacing) diff --git a/tests/fixtures/generic/mdadm/examine-raid1-resync b/tests/fixtures/generic/mdadm/examine-raid1-resync new file mode 100644 index 000000000..5c7c2aa5b --- /dev/null +++ b/tests/fixtures/generic/mdadm/examine-raid1-resync @@ -0,0 +1,28 @@ +/dev/md126: + Version : 1.2 + Creation Time : Sun Aug 7 21:15:49 2022 + Raid Level : raid1 + Array Size : 101376 (99.00 MiB 103.81 MB) + Used Dev Size : 101376 (99.00 MiB 103.81 MB) + Raid Devices : 2 + Total Devices : 2 + Persistence : Superblock is persistent + + Update Time : Sun Aug 7 21:25:52 2022 + State : clean, resyncing + Active Devices : 2 + Working Devices : 2 + Failed Devices : 0 + Spare Devices : 0 + +Consistency Policy : resync + + Resync Status : 27% complete + + Name : sysrescue:126 (local to host sysrescue) + UUID : e054553a:fbccdcfe:0ae80bc8:9379377a + Events : 20 + + Number Major Minor RaidDevice State + 0 254 1 0 active sync /dev/vda1 + 1 254 2 1 active sync /dev/vda2 diff --git a/tests/fixtures/generic/mdadm/examine-raid1-spare b/tests/fixtures/generic/mdadm/examine-raid1-spare new file mode 100644 index 000000000..97d43b00e --- /dev/null +++ b/tests/fixtures/generic/mdadm/examine-raid1-spare @@ -0,0 +1,26 @@ +/dev/vda4: + Magic : a92b4efc + Version : 1.2 + Feature Map : 0x0 + Array UUID : e054553a:fbccdcfe:0ae80bc8:9379377a + Name : sysrescue:126 (local to host sysrescue) + Creation Time : Sun Aug 7 21:15:49 2022 + Raid Level : raid1 + Raid Devices : 2 + + Avail Dev Size : 202752 sectors (99.00 MiB 103.81 MB) + Array Size : 101376 KiB (99.00 MiB 103.81 MB) + Data Offset : 2048 sectors + Super Offset : 8 sectors + Unused Space : before=1968 sectors, after=0 sectors + State : clean + Device UUID : 46b6f457:a3cf3db3:41217976:74a7243c + + Update Time : Sun Aug 7 21:39:58 2022 + Bad Block Log : 512 entries available at offset 16 sectors + Checksum : c265a5d4 - correct + Events : 52 + + + Device Role : spare + Array State : AA ('A' == active, '.' == missing, 'R' == replacing) diff --git a/tests/fixtures/generic/mdadm/examine-raid1-syncing b/tests/fixtures/generic/mdadm/examine-raid1-syncing new file mode 100644 index 000000000..0ae517b64 --- /dev/null +++ b/tests/fixtures/generic/mdadm/examine-raid1-syncing @@ -0,0 +1,26 @@ +/dev/vda1: + Magic : a92b4efc + Version : 1.2 + Feature Map : 0x0 + Array UUID : e054553a:fbccdcfe:0ae80bc8:9379377a + Name : sysrescue:126 (local to host sysrescue) + Creation Time : Sun Aug 7 21:15:49 2022 + Raid Level : raid1 + Raid Devices : 2 + + Avail Dev Size : 202752 sectors (99.00 MiB 103.81 MB) + Array Size : 101376 KiB (99.00 MiB 103.81 MB) + Data Offset : 2048 sectors + Super Offset : 8 sectors + Unused Space : before=1968 sectors, after=0 sectors + State : active + Device UUID : 1ec6a01d:e96d7f47:59970cf2:f0c056d2 + + Update Time : Sun Aug 7 21:15:49 2022 + Bad Block Log : 512 entries available at offset 16 sectors + Checksum : 2e1bdda4 - correct + Events : 0 + + + Device Role : Active device 0 + Array State : AA ('A' == active, '.' == missing, 'R' == replacing) diff --git a/tests/fixtures/generic/mdadm/mdadm-examine.out b/tests/fixtures/generic/mdadm/mdadm-examine.out new file mode 100644 index 000000000..c38826bf4 --- /dev/null +++ b/tests/fixtures/generic/mdadm/mdadm-examine.out @@ -0,0 +1,27 @@ +/dev/sdb1: + Magic : a92b4efc + Version : 1.1 + Feature Map : 0x1 + Array UUID : 85c5b164:d58a5ada:14f5fe07:d642e843 + Name : virttest:0 + Creation Time : Tue Apr 13 23:22:16 2010 + Raid Level : raid1 + Raid Devices : 2 + + Avail Dev Size : 11721041656 sectors (5.46 TiB 6.00 TB) + Array Size : 5860520828 KiB (5.46 TiB 6.00 TB) + Data Offset : 264 sectors + Super Offset : 0 sectors + Unused Space : before=80 sectors, after=0 sectors + State : clean + Device UUID : 813162e5:2e865efe:02ba5570:7003165c + +Internal Bitmap : 8 sectors from superblock + Update Time : Tue Jul 26 20:16:31 2022 + Bad Block Log : 512 entries available at offset 72 sectors + Checksum : f141a577 - correct + Events : 2193679 + + + Device Role : Active device 0 + Array State : AA ('A' == active, '.' == missing, 'R' == replacing) diff --git a/tests/fixtures/generic/mdadm/mdadm-query-detail.out b/tests/fixtures/generic/mdadm/mdadm-query-detail.out new file mode 100644 index 000000000..246f4ad78 --- /dev/null +++ b/tests/fixtures/generic/mdadm/mdadm-query-detail.out @@ -0,0 +1,28 @@ +/dev/md0: + Version : 1.1 + Creation Time : Tue Apr 13 23:22:16 2010 + Raid Level : raid1 + Array Size : 5860520828 (5.46 TiB 6.00 TB) + Used Dev Size : 5860520828 (5.46 TiB 6.00 TB) + Raid Devices : 2 + Total Devices : 2 + Persistence : Superblock is persistent + + Intent Bitmap : Internal + + Update Time : Tue Jul 26 20:16:31 2022 + State : clean + Active Devices : 2 + Working Devices : 2 + Failed Devices : 0 + Spare Devices : 0 + +Consistency Policy : bitmap + + Name : virttest:0 + UUID : 85c5b164:d58a5ada:14f5fe07:d642e843 + Events : 2193679 + + Number Major Minor RaidDevice State + 3 8 17 0 active sync /dev/sdb1 + 2 8 33 1 active sync /dev/sdc1 diff --git a/tests/fixtures/generic/mdadm/query-raid0-ok b/tests/fixtures/generic/mdadm/query-raid0-ok new file mode 100644 index 000000000..3aff3eb44 --- /dev/null +++ b/tests/fixtures/generic/mdadm/query-raid0-ok @@ -0,0 +1,28 @@ +/dev/md0: + Version : 1.2 + Creation Time : Sun Aug 7 20:56:52 2022 + Raid Level : raid0 + Array Size : 200704 (196.00 MiB 205.52 MB) + Raid Devices : 2 + Total Devices : 2 + Persistence : Superblock is persistent + + Update Time : Sun Aug 7 20:56:52 2022 + State : clean + Active Devices : 2 + Working Devices : 2 + Failed Devices : 0 + Spare Devices : 0 + + Layout : -unknown- + Chunk Size : 512K + +Consistency Policy : none + + Name : sysrescue:0 (local to host sysrescue) + UUID : 7e81a856:abb9c1c2:4b71237a:9778cc66 + Events : 0 + + Number Major Minor RaidDevice State + 0 254 1 0 active sync /dev/vda1 + 1 254 2 1 active sync /dev/vda2 diff --git a/tests/fixtures/generic/mdadm/query-raid1-failed-and-flags b/tests/fixtures/generic/mdadm/query-raid1-failed-and-flags new file mode 100644 index 000000000..e463b4d6e --- /dev/null +++ b/tests/fixtures/generic/mdadm/query-raid1-failed-and-flags @@ -0,0 +1,28 @@ +/dev/md127: + Version : 1.2 + Creation Time : Sun Aug 7 21:15:49 2022 + Raid Level : raid1 + Array Size : 101376 (99.00 MiB 103.81 MB) + Used Dev Size : 101376 (99.00 MiB 103.81 MB) + Raid Devices : 2 + Total Devices : 3 + Persistence : Superblock is persistent + + Update Time : Sun Aug 7 21:48:58 2022 + State : clean + Active Devices : 2 + Working Devices : 2 + Failed Devices : 1 + Spare Devices : 0 + +Consistency Policy : resync + + Name : sysrescue:126 (local to host sysrescue) + UUID : e054553a:fbccdcfe:0ae80bc8:9379377a + Events : 78 + + Number Major Minor RaidDevice State + 0 254 1 0 active sync /dev/vda1 + 2 254 2 1 active sync writemostly failfast /dev/vda2 + + 3 254 4 - faulty /dev/vda4 diff --git a/tests/fixtures/generic/mdadm/query-raid1-faulty b/tests/fixtures/generic/mdadm/query-raid1-faulty new file mode 100644 index 000000000..f001cfc16 --- /dev/null +++ b/tests/fixtures/generic/mdadm/query-raid1-faulty @@ -0,0 +1,28 @@ +/dev/md126: + Version : 1.2 + Creation Time : Sun Aug 7 21:15:49 2022 + Raid Level : raid1 + Array Size : 101376 (99.00 MiB 103.81 MB) + Used Dev Size : 101376 (99.00 MiB 103.81 MB) + Raid Devices : 2 + Total Devices : 2 + Persistence : Superblock is persistent + + Update Time : Sun Aug 7 21:28:34 2022 + State : clean, degraded + Active Devices : 1 + Working Devices : 1 + Failed Devices : 1 + Spare Devices : 0 + +Consistency Policy : resync + + Name : sysrescue:126 (local to host sysrescue) + UUID : e054553a:fbccdcfe:0ae80bc8:9379377a + Events : 25 + + Number Major Minor RaidDevice State + 0 254 1 0 active sync /dev/vda1 + - 0 0 1 removed + + 1 254 2 - faulty /dev/vda2 diff --git a/tests/fixtures/generic/mdadm/query-raid1-faulty-with-spare b/tests/fixtures/generic/mdadm/query-raid1-faulty-with-spare new file mode 100644 index 000000000..5df29f5ef --- /dev/null +++ b/tests/fixtures/generic/mdadm/query-raid1-faulty-with-spare @@ -0,0 +1,30 @@ +/dev/md127: + Version : 1.2 + Creation Time : Sun Aug 7 21:15:49 2022 + Raid Level : raid1 + Array Size : 101376 (99.00 MiB 103.81 MB) + Used Dev Size : 101376 (99.00 MiB 103.81 MB) + Raid Devices : 2 + Total Devices : 3 + Persistence : Superblock is persistent + + Update Time : Sun Aug 7 21:43:18 2022 + State : clean, degraded, recovering + Active Devices : 1 + Working Devices : 2 + Failed Devices : 1 + Spare Devices : 1 + +Consistency Policy : resync + + Rebuild Status : 24% complete + + Name : sysrescue:126 (local to host sysrescue) + UUID : e054553a:fbccdcfe:0ae80bc8:9379377a + Events : 57 + + Number Major Minor RaidDevice State + 0 254 1 0 active sync /dev/vda1 + 3 254 4 1 spare rebuilding /dev/vda4 + + 2 254 2 - faulty failfast /dev/vda2 diff --git a/tests/fixtures/generic/mdadm/query-raid1-faulty_and_removed b/tests/fixtures/generic/mdadm/query-raid1-faulty_and_removed new file mode 100644 index 000000000..b2a6f36e9 --- /dev/null +++ b/tests/fixtures/generic/mdadm/query-raid1-faulty_and_removed @@ -0,0 +1,26 @@ +/dev/md127: + Version : 1.2 + Creation Time : Sun Aug 7 21:15:49 2022 + Raid Level : raid1 + Array Size : 101376 (99.00 MiB 103.81 MB) + Used Dev Size : 101376 (99.00 MiB 103.81 MB) + Raid Devices : 2 + Total Devices : 1 + Persistence : Superblock is persistent + + Update Time : Sun Aug 7 21:32:27 2022 + State : clean, degraded + Active Devices : 1 + Working Devices : 1 + Failed Devices : 0 + Spare Devices : 0 + +Consistency Policy : resync + + Name : sysrescue:126 (local to host sysrescue) + UUID : e054553a:fbccdcfe:0ae80bc8:9379377a + Events : 31 + + Number Major Minor RaidDevice State + 0 254 1 0 active sync /dev/vda1 + - 0 0 1 removed diff --git a/tests/fixtures/generic/mdadm/query-raid1-ok b/tests/fixtures/generic/mdadm/query-raid1-ok new file mode 100644 index 000000000..704cee4b2 --- /dev/null +++ b/tests/fixtures/generic/mdadm/query-raid1-ok @@ -0,0 +1,26 @@ +/dev/md126: + Version : 1.2 + Creation Time : Sun Aug 7 21:15:49 2022 + Raid Level : raid1 + Array Size : 101376 (99.00 MiB 103.81 MB) + Used Dev Size : 101376 (99.00 MiB 103.81 MB) + Raid Devices : 2 + Total Devices : 2 + Persistence : Superblock is persistent + + Update Time : Sun Aug 7 21:20:08 2022 + State : clean + Active Devices : 2 + Working Devices : 2 + Failed Devices : 0 + Spare Devices : 0 + +Consistency Policy : resync + + Name : sysrescue:126 (local to host sysrescue) + UUID : e054553a:fbccdcfe:0ae80bc8:9379377a + Events : 17 + + Number Major Minor RaidDevice State + 0 254 1 0 active sync /dev/vda1 + 1 254 2 1 active sync /dev/vda2 diff --git a/tests/fixtures/generic/mdadm/query-raid1-ok-0.9 b/tests/fixtures/generic/mdadm/query-raid1-ok-0.9 new file mode 100644 index 000000000..617593b40 --- /dev/null +++ b/tests/fixtures/generic/mdadm/query-raid1-ok-0.9 @@ -0,0 +1,26 @@ +/dev/md0: + Version : 0.90 + Creation Time : Sun Aug 7 21:52:56 2022 + Raid Level : raid1 + Array Size : 102336 (99.94 MiB 104.79 MB) + Used Dev Size : 102336 (99.94 MiB 104.79 MB) + Raid Devices : 2 + Total Devices : 2 + Preferred Minor : 0 + Persistence : Superblock is persistent + + Update Time : Sun Aug 7 21:54:35 2022 + State : clean + Active Devices : 2 + Working Devices : 2 + Failed Devices : 0 + Spare Devices : 0 + +Consistency Policy : resync + + UUID : 3f720601:29af5283:96fc04a8:108a4af7 (local to host sysrescue) + Events : 0.18 + + Number Major Minor RaidDevice State + 0 254 1 0 active sync /dev/vda1 + 1 254 2 1 active sync /dev/vda2 diff --git a/tests/fixtures/generic/mdadm/query-raid1-ok-failfast b/tests/fixtures/generic/mdadm/query-raid1-ok-failfast new file mode 100644 index 000000000..18d8adc64 --- /dev/null +++ b/tests/fixtures/generic/mdadm/query-raid1-ok-failfast @@ -0,0 +1,26 @@ +/dev/md127: + Version : 1.2 + Creation Time : Sun Aug 7 21:15:49 2022 + Raid Level : raid1 + Array Size : 101376 (99.00 MiB 103.81 MB) + Used Dev Size : 101376 (99.00 MiB 103.81 MB) + Raid Devices : 2 + Total Devices : 2 + Persistence : Superblock is persistent + + Update Time : Sun Aug 7 21:37:34 2022 + State : clean + Active Devices : 2 + Working Devices : 2 + Failed Devices : 0 + Spare Devices : 0 + +Consistency Policy : resync + + Name : sysrescue:126 (local to host sysrescue) + UUID : e054553a:fbccdcfe:0ae80bc8:9379377a + Events : 51 + + Number Major Minor RaidDevice State + 0 254 1 0 active sync /dev/vda1 + 2 254 2 1 active sync failfast /dev/vda2 diff --git a/tests/fixtures/generic/mdadm/query-raid1-ok-spare b/tests/fixtures/generic/mdadm/query-raid1-ok-spare new file mode 100644 index 000000000..3edb9699e --- /dev/null +++ b/tests/fixtures/generic/mdadm/query-raid1-ok-spare @@ -0,0 +1,28 @@ +/dev/md127: + Version : 1.2 + Creation Time : Sun Aug 7 21:15:49 2022 + Raid Level : raid1 + Array Size : 101376 (99.00 MiB 103.81 MB) + Used Dev Size : 101376 (99.00 MiB 103.81 MB) + Raid Devices : 2 + Total Devices : 3 + Persistence : Superblock is persistent + + Update Time : Sun Aug 7 21:39:58 2022 + State : clean + Active Devices : 2 + Working Devices : 3 + Failed Devices : 0 + Spare Devices : 1 + +Consistency Policy : resync + + Name : sysrescue:126 (local to host sysrescue) + UUID : e054553a:fbccdcfe:0ae80bc8:9379377a + Events : 52 + + Number Major Minor RaidDevice State + 0 254 1 0 active sync /dev/vda1 + 2 254 2 1 active sync failfast /dev/vda2 + + 3 254 4 - spare /dev/vda4 diff --git a/tests/fixtures/generic/mdadm/query-raid1-rebuild-failfast b/tests/fixtures/generic/mdadm/query-raid1-rebuild-failfast new file mode 100644 index 000000000..9b3d42ac4 --- /dev/null +++ b/tests/fixtures/generic/mdadm/query-raid1-rebuild-failfast @@ -0,0 +1,28 @@ +/dev/md127: + Version : 1.2 + Creation Time : Sun Aug 7 21:15:49 2022 + Raid Level : raid1 + Array Size : 101376 (99.00 MiB 103.81 MB) + Used Dev Size : 101376 (99.00 MiB 103.81 MB) + Raid Devices : 2 + Total Devices : 2 + Persistence : Superblock is persistent + + Update Time : Sun Aug 7 21:36:17 2022 + State : clean, degraded, recovering + Active Devices : 1 + Working Devices : 2 + Failed Devices : 0 + Spare Devices : 1 + +Consistency Policy : resync + + Rebuild Status : 20% complete + + Name : sysrescue:126 (local to host sysrescue) + UUID : e054553a:fbccdcfe:0ae80bc8:9379377a + Events : 37 + + Number Major Minor RaidDevice State + 0 254 1 0 active sync /dev/vda1 + 2 254 2 1 failfast spare rebuilding /dev/vda2 diff --git a/tests/fixtures/generic/mdadm/query-raid1-spare-writem-rebuild b/tests/fixtures/generic/mdadm/query-raid1-spare-writem-rebuild new file mode 100644 index 000000000..3e75eed12 --- /dev/null +++ b/tests/fixtures/generic/mdadm/query-raid1-spare-writem-rebuild @@ -0,0 +1,29 @@ +/dev/md127: + Version : 1.2 + Creation Time : Sun Aug 7 21:15:49 2022 + Raid Level : raid1 + Array Size : 101376 (99.00 MiB 103.81 MB) + Used Dev Size : 101376 (99.00 MiB 103.81 MB) + Raid Devices : 2 + Total Devices : 3 + Persistence : Superblock is persistent + + Update Time : Sun Aug 7 21:47:42 2022 + State : clean, recovering + Active Devices : 2 + Working Devices : 3 + Failed Devices : 0 + Spare Devices : 1 + +Consistency Policy : resync + + Rebuild Status : 30% complete + + Name : sysrescue:126 (local to host sysrescue) + UUID : e054553a:fbccdcfe:0ae80bc8:9379377a + Events : 74 + + Number Major Minor RaidDevice State + 0 254 1 0 active sync /dev/vda1 + 2 254 2 1 writemostly failfast spare rebuilding /dev/vda2 + 3 254 4 1 active sync /dev/vda4 diff --git a/tests/fixtures/generic/mdadm/query-raid1-syncing b/tests/fixtures/generic/mdadm/query-raid1-syncing new file mode 100644 index 000000000..968cb4099 --- /dev/null +++ b/tests/fixtures/generic/mdadm/query-raid1-syncing @@ -0,0 +1,28 @@ +/dev/md126: + Version : 1.2 + Creation Time : Sun Aug 7 21:15:49 2022 + Raid Level : raid1 + Array Size : 101376 (99.00 MiB 103.81 MB) + Used Dev Size : 101376 (99.00 MiB 103.81 MB) + Raid Devices : 2 + Total Devices : 2 + Persistence : Superblock is persistent + + Update Time : Sun Aug 7 21:15:49 2022 + State : clean, resyncing + Active Devices : 2 + Working Devices : 2 + Failed Devices : 0 + Spare Devices : 0 + +Consistency Policy : resync + + Resync Status : 4% complete + + Name : sysrescue:126 (local to host sysrescue) + UUID : e054553a:fbccdcfe:0ae80bc8:9379377a + Events : 0 + + Number Major Minor RaidDevice State + 0 254 1 0 active sync /dev/vda1 + 1 254 2 1 active sync /dev/vda2 From b7a281c6c46c49bfcb5c213709f02b4f4bd454f5 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sun, 7 Aug 2022 15:11:16 -0700 Subject: [PATCH 052/116] add val fields and append device table flags to state --- jc/parsers/mdadm.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/jc/parsers/mdadm.py b/jc/parsers/mdadm.py index 874cbae6c..b26df2857 100644 --- a/jc/parsers/mdadm.py +++ b/jc/parsers/mdadm.py @@ -67,7 +67,8 @@ def _process(proc_data: Dict) -> Dict: 'active_devices', 'working_devices', 'failed_devices', 'spare_devices', 'events', 'number', 'major', 'minor', 'raid_device', 'avail_dev_size_num', 'data_offset', 'super_offset', 'unused_space_before', 'unused_space_after', - 'chunk_size', 'preferred_minor', 'check_status_percent', 'resync_status_percent'} + 'chunk_size', 'preferred_minor', 'check_status_percent', 'resync_status_percent', + 'rebuild_status_percent'} array_state_map = { 'A': 'active', @@ -103,6 +104,9 @@ def _process(proc_data: Dict) -> Dict: if 'name' in proc_data: proc_data['name_val'] = proc_data['name'].split(maxsplit=1)[0] + if 'uuid' in proc_data: + proc_data['uuid_val'] = proc_data['uuid'].split(maxsplit=1)[0] + if 'checksum' in proc_data: proc_data['checksum_val'] = proc_data['checksum'].split(maxsplit=1)[0] proc_data['checksum_state'] = proc_data['checksum'].split()[-1] @@ -129,6 +133,9 @@ def _process(proc_data: Dict) -> Dict: if 'check_status' in proc_data: proc_data['check_status_percent'] = proc_data['check_status'].split('%')[0] + if 'rebuild_status' in proc_data: + proc_data['rebuild_status_percent'] = proc_data['rebuild_status'].split('%')[0] + # add timestamp fields if 'creation_time' in proc_data: dt = jc.utils.timestamp(proc_data['creation_time'], format_hint=(1000,)) @@ -215,8 +222,12 @@ def parse( if 'RaidDevice' in item: item['raid_device'] = item.pop('RaidDevice') - raw_output['device_table'] = d_table - + if 'device' in item and item['device']: + if not item['device'].startswith('/'): + flags, dev = item['device'].rsplit(maxsplit=1) + item['device'] = dev + item['state'] = item['state'] + ' ' + flags + raw_output['device_table'] = d_table # type: ignore return raw_output if raw else _process(raw_output) From b087e712ca1199b7270a73b995443a218043be5e Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Tue, 9 Aug 2022 20:25:54 -0700 Subject: [PATCH 053/116] initial cef parser --- CHANGELOG | 1 + jc/lib.py | 1 + jc/parsers/cef.py | 226 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 228 insertions(+) create mode 100644 jc/parsers/cef.py diff --git a/CHANGELOG b/CHANGELOG index a7d7cf562..2ee8d2e58 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -2,6 +2,7 @@ jc changelog xxxxxxxx v1.20.5 - Add IP Address string parser +- Add CEF string parser - Add PLIST file parser (XML and binary support) - Add `mdadm` command parser tested on linux (IN PROGRESS) - Add `--time-out` or `-t` option to add a UTC timestamp to the JSON output diff --git a/jc/lib.py b/jc/lib.py index 40314c829..955b4706c 100644 --- a/jc/lib.py +++ b/jc/lib.py @@ -16,6 +16,7 @@ 'asciitable', 'asciitable-m', 'blkid', + 'cef', 'chage', 'cksum', 'crontab', diff --git a/jc/parsers/cef.py b/jc/parsers/cef.py new file mode 100644 index 000000000..c3a539412 --- /dev/null +++ b/jc/parsers/cef.py @@ -0,0 +1,226 @@ +"""jc - JSON Convert CEF string parser + +This is a best-effort parser since there are so many variations to CEF +formatting from different vendors. If you require special handling for your +CEF input, you can copy this parser code to the `jc` pluggin directory for +your system and modify it to suit your needs. + +This parser will accept a single CEF string or multiple CEF string lines. +Any text before "CEF" will be ignored. + +Usage (cli): + + $ echo 'CEF:0|Vendor|Product|3.2.0|1|SYSTEM|1|... | jc --cef + +Usage (module): + + import jc + result = jc.parse('cef', cef_string_output) + +Schema: + + [ + { + "cef": string, + "bar": boolean, + "baz": integer + } + ] + +Examples: + + $ cef | jc --cef -p + [] + + $ cef | jc --cef -p -r + [] +""" +from typing import List, Dict +import re +import jc.utils +from jc.exceptions import ParseError + + +class info(): + """Provides parser metadata (version, author, etc.)""" + version = '1.0' + description = 'CEF string parser' + author = 'Kelly Brazil' + author_email = 'kellyjonbrazil@gmail.com' + details = 'Using the pycef library at https://github.com/DavidJBianco/pycef/releases/tag/v1.11-2' + compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd'] + +__version__ = info.version + + +############################################################################ +""" +The MIT License (MIT) + +Copyright (c) 2016 DavidJBianco + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +def _pycef_parse(str_input): + """ + Parse a string in CEF format and return a dict with the header values + and the extension data. + """ + + # Create the empty dict we'll return later + values = dict() + + # This regex separates the string into the CEF header and the extension + # data. Once we do this, it's easier to use other regexes to parse each + # part. + header_re = r'((CEF:\d+)([^=\\]+\|){,7})(.*)' + + res = re.search(header_re, str_input) + + if res: + header = res.group(1) + extension = res.group(4) + + # Split the header on the "|" char. Uses a negative lookbehind + # assertion to ensure we don't accidentally split on escaped chars, + # though. + spl = re.split(r'(? 6: + values["Severity"] = spl[6] + + # The first value is actually the CEF version, formatted like + # "CEF:#". Ignore anything before that (like a date from a syslog message). + # We then split on the colon and use the second value as the + # version number. + cef_start = spl[0].find('CEF') + if cef_start == -1: + raise ParseError('Invalid CEF string.') + (cef, version) = spl[0][cef_start:].split(':') + values["CEFVersion"] = version + + # The ugly, gnarly regex here finds a single key=value pair, + # taking into account multiple whitespaces, escaped '=' and '|' + # chars. It returns an iterator of tuples. + spl = re.findall(r'([^=\s]+)=((?:[\\]=|[^=])+)(?:\s|$)', extension) + + for i in spl: + # Split the tuples and put them into the dictionary + values[i[0]] = i[1] + + # Process custom field labels + for key in list(values.keys()): + # If the key string ends with Label, replace it in the appropriate + # custom field + if key[-5:] == "Label": + customlabel = key[:-5] + # Find the corresponding customfield and replace with the label + for customfield in list(values.keys()): + if customfield == customlabel: + values[values[key]] = values[customfield] + del values[customfield] + del values[key] + else: + raise ParseError('Could not parse record. Is it valid CEF format?') + + return values + +############################################################################ + + +def _process(proc_data: List[Dict]) -> List[Dict]: + """ + Final processing to conform to the schema. + + Parameters: + + proc_data: (List of Dictionaries) raw structured data to process + + Returns: + + List of Dictionaries. Structured to conform to the schema. + """ + # fix escape chars specified in syslog RFC 5424 + # https://www.rfc-editor.org/rfc/rfc5424.html#section-6 + escape_map = { + r'\\': '\\', + r'\"': r'"', + r'\]': r']' + } + + for item in proc_data: + for key, value in item.copy().items(): + # remove any spaces around values + item[key] = value.strip() + + # fixup escaped characters + for esc, esc_sub in escape_map.items(): + item[key] = item[key].replace(esc, esc_sub) + + # remove any quotation marks from key names + if '"' in key: + new_key = key.replace('"', '') + item[new_key] = item.pop(key) + + return proc_data + + +def parse( + data: str, + raw: bool = False, + quiet: bool = False +) -> List[Dict]: + """ + Main text parsing function + + Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + + Returns: + + List of Dictionaries. Raw or processed structured data. + """ + jc.utils.compatibility(__name__, info.compatible, quiet) + jc.utils.input_type_check(data) + + raw_output: List = [] + + if jc.utils.has_data(data): + for line in filter(None, data.splitlines()): + raw_output.append(_pycef_parse(line)) + + return raw_output if raw else _process(raw_output) From 345431bbcc7b660c16aa0ca1fa567a3c48ace2ee Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Tue, 9 Aug 2022 21:33:54 -0700 Subject: [PATCH 054/116] add more escape chars --- jc/parsers/cef.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/jc/parsers/cef.py b/jc/parsers/cef.py index c3a539412..db89f53b7 100644 --- a/jc/parsers/cef.py +++ b/jc/parsers/cef.py @@ -6,7 +6,9 @@ your system and modify it to suit your needs. This parser will accept a single CEF string or multiple CEF string lines. -Any text before "CEF" will be ignored. +Any text before "CEF" will be ignored. Syslog and CEF escaped characters +(`\\`, `\\"`, `\\]`, `\\|`, `\\n`, `\\r`) are unescaped. To preserve +escaping, use the `--raw` or `raw=True` option in the `parse()` function. Usage (cli): @@ -171,12 +173,16 @@ def _process(proc_data: List[Dict]) -> List[Dict]: List of Dictionaries. Structured to conform to the schema. """ - # fix escape chars specified in syslog RFC 5424 + # fix escape chars specified in syslog RFC 5424 and CEF spec # https://www.rfc-editor.org/rfc/rfc5424.html#section-6 escape_map = { r'\\': '\\', r'\"': r'"', - r'\]': r']' + r'\]': r']', + r'\|': r'|', + r'\=': r'=', + r'\n': '\n', + r'\r': '\r' } for item in proc_data: From edff49a44ff38589a884042fe9ab930ef212a543 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 11 Aug 2022 13:29:53 -0700 Subject: [PATCH 055/116] add syslog parsers --- CHANGELOG | 1 + jc/lib.py | 2 + jc/parsers/syslog_3164.py | 126 +++++++++++++++++++++++++++++++++ jc/parsers/syslog_5424.py | 142 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 271 insertions(+) create mode 100644 jc/parsers/syslog_3164.py create mode 100644 jc/parsers/syslog_5424.py diff --git a/CHANGELOG b/CHANGELOG index 2ee8d2e58..f60202468 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -2,6 +2,7 @@ jc changelog xxxxxxxx v1.20.5 - Add IP Address string parser +- Add Syslog string parsers (RFC 3164 and RFC 5424) - Add CEF string parser - Add PLIST file parser (XML and binary support) - Add `mdadm` command parser tested on linux (IN PROGRESS) diff --git a/jc/lib.py b/jc/lib.py index 955b4706c..1c2a07973 100644 --- a/jc/lib.py +++ b/jc/lib.py @@ -94,6 +94,8 @@ 'stat', 'stat-s', 'sysctl', + 'syslog-3164', + 'syslog-5424', 'systemctl', 'systemctl-lj', 'systemctl-ls', diff --git a/jc/parsers/syslog_3164.py b/jc/parsers/syslog_3164.py new file mode 100644 index 000000000..dd6834b23 --- /dev/null +++ b/jc/parsers/syslog_3164.py @@ -0,0 +1,126 @@ +"""jc - JSON Convert Syslog RFC 3164 string parser + +<> + +Usage (cli): + + $ syslog-3164 | jc --syslog-3164 + + or + + $ jc syslog-3164 + +Usage (module): + + import jc + result = jc.parse('syslog_3164', syslog_command_output) + +Schema: + + [ + { + "syslog-3164": string, + "bar": boolean, + "baz": integer + } + ] + +Examples: + + $ syslog-3164 | jc --syslog-3164 -p + [] + + $ syslog-3164 | jc --syslog-3164 -p -r + [] +""" +import re +from typing import List, Dict +import jc.utils + + +class info(): + """Provides parser metadata (version, author, etc.)""" + version = '1.0' + description = 'Syslog RFC 3164 string parser' + author = 'Kelly Brazil' + author_email = 'kellyjonbrazil@gmail.com' + compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd'] + +__version__ = info.version + + +def _process(proc_data: List[Dict]) -> List[Dict]: + """ + Final processing to conform to the schema. + + Parameters: + + proc_data: (List of Dictionaries) raw structured data to process + + Returns: + + List of Dictionaries. Structured to conform to the schema. + """ + + # process the data here + # rebuild output for added semantic information + # use helper functions in jc.utils for int, float, bool + # conversions and timestamps + + return proc_data + + +def parse( + data: str, + raw: bool = False, + quiet: bool = False +) -> List[Dict]: + """ + Main text parsing function + + Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + + Returns: + + List of Dictionaries. Raw or processed structured data. + """ + jc.utils.compatibility(__name__, info.compatible, quiet) + jc.utils.input_type_check(data) + + raw_output: List = [] + + # inspired by https://gist.github.com/miticojo/b16bb13e78572c2d2fac82d9516d5c32 + syslog = re.compile(r''' + (?P<\d*>)? + (?P[A-Z][a-z][a-z]\s{1,2}\d{1,2}\s\d{2}?:\d{2}:\d{2})\s + (?P[\w][\w\d\.@-]*)\s + (?P[\w\d\[\]\.@-]+):?\s + (?P.*) + ''', re.VERBOSE + ) + + if jc.utils.has_data(data): + + for line in filter(None, data.splitlines()): + syslog_match = syslog.match(line) + if syslog_match: + priority = None + if syslog_match.group('priority'): + priority = syslog_match.group('priority')[1:-1] + + syslog_dict = { + 'priority': priority, + 'date': syslog_match.group('date'), + 'hostname': syslog_match.group('host'), + 'tag': syslog_match.group('tag'), + 'message': syslog_match.group('message') + } + + if syslog_dict: + raw_output.append(syslog_dict) + + return raw_output if raw else _process(raw_output) diff --git a/jc/parsers/syslog_5424.py b/jc/parsers/syslog_5424.py new file mode 100644 index 000000000..9fa646a22 --- /dev/null +++ b/jc/parsers/syslog_5424.py @@ -0,0 +1,142 @@ +"""jc - JSON Convert Syslog RFC 5424 string parser + +<> + +Usage (cli): + + $ syslog-5424 | jc --syslog-5424 + + or + + $ jc syslog-5424 + +Usage (module): + + import jc + result = jc.parse('syslog_5424', syslog_command_output) + +Schema: + + [ + { + "syslog-5424": string, + "bar": boolean, + "baz": integer + } + ] + +Examples: + + $ syslog-5424 | jc --syslog-5424 -p + [] + + $ syslog-5424 | jc --syslog-5424 -p -r + [] +""" +import re +from typing import List, Dict +import jc.utils + + +class info(): + """Provides parser metadata (version, author, etc.)""" + version = '1.0' + description = 'Syslog RFC 5424 string parser' + author = 'Kelly Brazil' + author_email = 'kellyjonbrazil@gmail.com' + compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd'] + +__version__ = info.version + + +def _process(proc_data: List[Dict]) -> List[Dict]: + """ + Final processing to conform to the schema. + + Parameters: + + proc_data: (List of Dictionaries) raw structured data to process + + Returns: + + List of Dictionaries. Structured to conform to the schema. + """ + + # process the data here + # rebuild output for added semantic information + # use helper functions in jc.utils for int, float, bool + # conversions and timestamps + + return proc_data + + +def parse( + data: str, + raw: bool = False, + quiet: bool = False +) -> List[Dict]: + """ + Main text parsing function + + Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + + Returns: + + List of Dictionaries. Raw or processed structured data. + """ + jc.utils.compatibility(__name__, info.compatible, quiet) + jc.utils.input_type_check(data) + + raw_output: List = [] + syslog_dict = {} + + # inspired by https://regex101.com/library/Wgbxn2 + syslog = re.compile(r''' + (?P<(\d|\d{2}|1[1-8]\d|19[01])>)? + (?P\d{1,2})?\s* + (?P-|(?P[12]\d{3})- + (?P0\d|[1][012])- + (?P[012]\d|3[01])T + (?P[01]\d|2[0-4]): + (?P[0-5]\d): + (?P[0-5]\d|60)(?#60seconds can be used for leap year!)(?:\. + (?P\d{1,6}))? + (?PZ|[+-]\d{2}:\d{2})(?#=timezone))\s + (?P[\S]{1,255})\s + (?P[\S]{1,48})\s + (?P[\S]{1,128})\s + (?P[\S]{1,32})\s + (?P-|(?:\[.+?(?.+))? + ''', re.VERBOSE + ) + + if jc.utils.has_data(data): + + for line in filter(None, data.splitlines()): + syslog_match = syslog.match(line) + if syslog_match: + priority = None + if syslog_match.group('priority'): + priority = syslog_match.group('priority')[1:-1] + + syslog_dict = { + 'priority': priority, + 'version': syslog_match.group('version'), + 'timestamp': syslog_match.group('timestamp'), + 'hostname': syslog_match.group('hostname'), + 'appname': syslog_match.group('appname'), + 'proc_id': syslog_match.group('procid'), + 'msg_id': syslog_match.group('msgid'), + 'struct': syslog_match.group('structureddata'), + 'message': syslog_match.group('msg') + } + + if syslog_dict: + raw_output.append(syslog_dict) + + return raw_output if raw else _process(raw_output) From 5f280c3e245dd5c2c08c73c638c8935a1cdeaf8f Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 12 Aug 2022 10:16:15 -0700 Subject: [PATCH 056/116] doc update --- README.md | 4 + completions/jc_bash_completion.sh | 4 +- completions/jc_zsh_completion.sh | 9 ++- docs/parsers/cef.md | 68 +++++++++++++++++ docs/parsers/mdadm.md | 64 ++++++++++++++++ docs/parsers/syslog.md | 64 ++++++++++++++++ docs/parsers/syslog_bsd.md | 64 ++++++++++++++++ jc/lib.py | 4 +- jc/parsers/{syslog_5424.py => syslog.py} | 80 +++++++++++++++----- jc/parsers/{syslog_3164.py => syslog_bsd.py} | 4 +- man/jc.1 | 22 +++++- 11 files changed, 358 insertions(+), 29 deletions(-) create mode 100644 docs/parsers/cef.md create mode 100644 docs/parsers/mdadm.md create mode 100644 docs/parsers/syslog.md create mode 100644 docs/parsers/syslog_bsd.md rename jc/parsers/{syslog_5424.py => syslog.py} (55%) rename jc/parsers/{syslog_3164.py => syslog_bsd.py} (96%) diff --git a/README.md b/README.md index 94df9da5a..98ab3a51e 100644 --- a/README.md +++ b/README.md @@ -157,6 +157,7 @@ option. | ` --asciitable` | ASCII and Unicode table parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/asciitable) | | ` --asciitable-m` | multi-line ASCII and Unicode table parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/asciitable_m) | | ` --blkid` | `blkid` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/blkid) | +| ` --cef` | CEF string parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/cef) | | ` --chage` | `chage --list` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/chage) | | ` --cksum` | `cksum` and `sum` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/cksum) | | ` --crontab` | `crontab` command and file parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/crontab) | @@ -207,6 +208,7 @@ option. | ` --lsof` | `lsof` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/lsof) | | ` --lsusb` | `lsusb` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/lsusb) | | ` --m3u` | M3U and M3U8 file parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/m3u) | +| ` --mdadm` | `mdadm` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/mdadm) | | ` --mount` | `mount` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/mount) | | ` --mpstat` | `mpstat` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/mpstat) | | ` --mpstat-s` | `mpstat` command streaming parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/mpstat_s) | @@ -233,6 +235,8 @@ option. | ` --stat` | `stat` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/stat) | | ` --stat-s` | `stat` command streaming parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/stat_s) | | ` --sysctl` | `sysctl` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/sysctl) | +| ` --syslog-bsd` | Syslog RFC 3164 string parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/syslog_bsd) | +| ` --syslog` | Syslog RFC 5424 string parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/syslog) | | ` --systemctl` | `systemctl` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/systemctl) | | ` --systemctl-lj` | `systemctl list-jobs` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/systemctl_lj) | | ` --systemctl-ls` | `systemctl list-sockets` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/systemctl_ls) | diff --git a/completions/jc_bash_completion.sh b/completions/jc_bash_completion.sh index b1230cc8a..1e9ba714a 100644 --- a/completions/jc_bash_completion.sh +++ b/completions/jc_bash_completion.sh @@ -3,8 +3,8 @@ _jc() local cur prev words cword jc_commands jc_parsers jc_options \ jc_about_options jc_about_mod_options jc_help_options jc_special_options - jc_commands=(acpi airport arp blkid chage cksum crontab date df dig dmidecode dpkg du env file finger free git gpg hciconfig id ifconfig iostat iptables iw jobs last lastb ls lsblk lsmod lsof lsusb md5 md5sum mount mpstat netstat nmcli ntpq pidstat ping ping6 pip pip3 postconf printenv ps route rpm rsync sfdisk sha1sum sha224sum sha256sum sha384sum sha512sum shasum ss stat sum sysctl systemctl systeminfo timedatectl top tracepath tracepath6 traceroute traceroute6 ufw uname update-alternatives upower uptime vdir vmstat w wc who xrandr zipinfo) - jc_parsers=(--acpi --airport --airport-s --arp --asciitable --asciitable-m --blkid --chage --cksum --crontab --crontab-u --csv --csv-s --date --df --dig --dir --dmidecode --dpkg-l --du --email-address --env --file --finger --free --fstab --git-log --git-log-s --gpg --group --gshadow --hash --hashsum --hciconfig --history --hosts --id --ifconfig --ini --iostat --iostat-s --ip-address --iptables --iso-datetime --iw-scan --jar-manifest --jobs --jwt --kv --last --ls --ls-s --lsblk --lsmod --lsof --lsusb --m3u --mount --mpstat --mpstat-s --netstat --nmcli --ntpq --passwd --pidstat --pidstat-s --ping --ping-s --pip-list --pip-show --plist --postconf --ps --route --rpm-qi --rsync --rsync-s --sfdisk --shadow --ss --stat --stat-s --sysctl --systemctl --systemctl-lj --systemctl-ls --systemctl-luf --systeminfo --time --timedatectl --timestamp --top --top-s --tracepath --traceroute --ufw --ufw-appinfo --uname --update-alt-gs --update-alt-q --upower --uptime --url --vmstat --vmstat-s --w --wc --who --x509-cert --xml --xrandr --yaml --zipinfo) + jc_commands=(acpi airport arp blkid chage cksum crontab date df dig dmidecode dpkg du env file finger free git gpg hciconfig id ifconfig iostat iptables iw jobs last lastb ls lsblk lsmod lsof lsusb md5 md5sum mdadm mount mpstat netstat nmcli ntpq pidstat ping ping6 pip pip3 postconf printenv ps route rpm rsync sfdisk sha1sum sha224sum sha256sum sha384sum sha512sum shasum ss stat sum sysctl systemctl systeminfo timedatectl top tracepath tracepath6 traceroute traceroute6 ufw uname update-alternatives upower uptime vdir vmstat w wc who xrandr zipinfo) + jc_parsers=(--acpi --airport --airport-s --arp --asciitable --asciitable-m --blkid --cef --chage --cksum --crontab --crontab-u --csv --csv-s --date --df --dig --dir --dmidecode --dpkg-l --du --email-address --env --file --finger --free --fstab --git-log --git-log-s --gpg --group --gshadow --hash --hashsum --hciconfig --history --hosts --id --ifconfig --ini --iostat --iostat-s --ip-address --iptables --iso-datetime --iw-scan --jar-manifest --jobs --jwt --kv --last --ls --ls-s --lsblk --lsmod --lsof --lsusb --m3u --mdadm --mount --mpstat --mpstat-s --netstat --nmcli --ntpq --passwd --pidstat --pidstat-s --ping --ping-s --pip-list --pip-show --plist --postconf --ps --route --rpm-qi --rsync --rsync-s --sfdisk --shadow --ss --stat --stat-s --sysctl --syslog-bsd --syslog --systemctl --systemctl-lj --systemctl-ls --systemctl-luf --systeminfo --time --timedatectl --timestamp --top --top-s --tracepath --traceroute --ufw --ufw-appinfo --uname --update-alt-gs --update-alt-q --upower --uptime --url --vmstat --vmstat-s --w --wc --who --x509-cert --xml --xrandr --yaml --zipinfo) jc_options=(--force-color -C --debug -d --monochrome -m --pretty -p --quiet -q --raw -r --time-out -t --unbuffer -u --yaml-out -y) jc_about_options=(--about -a) jc_about_mod_options=(--pretty -p --yaml-out -y --monochrome -m --force-color -C) diff --git a/completions/jc_zsh_completion.sh b/completions/jc_zsh_completion.sh index 153489dfe..60b2ee09e 100644 --- a/completions/jc_zsh_completion.sh +++ b/completions/jc_zsh_completion.sh @@ -9,7 +9,7 @@ _jc() { jc_help_options jc_help_options_describe \ jc_special_options jc_special_options_describe - jc_commands=(acpi airport arp blkid chage cksum crontab date df dig dmidecode dpkg du env file finger free git gpg hciconfig id ifconfig iostat iptables iw jobs last lastb ls lsblk lsmod lsof lsusb md5 md5sum mount mpstat netstat nmcli ntpq pidstat ping ping6 pip pip3 postconf printenv ps route rpm rsync sfdisk sha1sum sha224sum sha256sum sha384sum sha512sum shasum ss stat sum sysctl systemctl systeminfo timedatectl top tracepath tracepath6 traceroute traceroute6 ufw uname update-alternatives upower uptime vdir vmstat w wc who xrandr zipinfo) + jc_commands=(acpi airport arp blkid chage cksum crontab date df dig dmidecode dpkg du env file finger free git gpg hciconfig id ifconfig iostat iptables iw jobs last lastb ls lsblk lsmod lsof lsusb md5 md5sum mdadm mount mpstat netstat nmcli ntpq pidstat ping ping6 pip pip3 postconf printenv ps route rpm rsync sfdisk sha1sum sha224sum sha256sum sha384sum sha512sum shasum ss stat sum sysctl systemctl systeminfo timedatectl top tracepath tracepath6 traceroute traceroute6 ufw uname update-alternatives upower uptime vdir vmstat w wc who xrandr zipinfo) jc_commands_describe=( 'acpi:run "acpi" command with magic syntax.' 'airport:run "airport" command with magic syntax.' @@ -46,6 +46,7 @@ _jc() { 'lsusb:run "lsusb" command with magic syntax.' 'md5:run "md5" command with magic syntax.' 'md5sum:run "md5sum" command with magic syntax.' + 'mdadm:run "mdadm" command with magic syntax.' 'mount:run "mount" command with magic syntax.' 'mpstat:run "mpstat" command with magic syntax.' 'netstat:run "netstat" command with magic syntax.' @@ -94,7 +95,7 @@ _jc() { 'xrandr:run "xrandr" command with magic syntax.' 'zipinfo:run "zipinfo" command with magic syntax.' ) - jc_parsers=(--acpi --airport --airport-s --arp --asciitable --asciitable-m --blkid --chage --cksum --crontab --crontab-u --csv --csv-s --date --df --dig --dir --dmidecode --dpkg-l --du --email-address --env --file --finger --free --fstab --git-log --git-log-s --gpg --group --gshadow --hash --hashsum --hciconfig --history --hosts --id --ifconfig --ini --iostat --iostat-s --ip-address --iptables --iso-datetime --iw-scan --jar-manifest --jobs --jwt --kv --last --ls --ls-s --lsblk --lsmod --lsof --lsusb --m3u --mount --mpstat --mpstat-s --netstat --nmcli --ntpq --passwd --pidstat --pidstat-s --ping --ping-s --pip-list --pip-show --plist --postconf --ps --route --rpm-qi --rsync --rsync-s --sfdisk --shadow --ss --stat --stat-s --sysctl --systemctl --systemctl-lj --systemctl-ls --systemctl-luf --systeminfo --time --timedatectl --timestamp --top --top-s --tracepath --traceroute --ufw --ufw-appinfo --uname --update-alt-gs --update-alt-q --upower --uptime --url --vmstat --vmstat-s --w --wc --who --x509-cert --xml --xrandr --yaml --zipinfo) + jc_parsers=(--acpi --airport --airport-s --arp --asciitable --asciitable-m --blkid --cef --chage --cksum --crontab --crontab-u --csv --csv-s --date --df --dig --dir --dmidecode --dpkg-l --du --email-address --env --file --finger --free --fstab --git-log --git-log-s --gpg --group --gshadow --hash --hashsum --hciconfig --history --hosts --id --ifconfig --ini --iostat --iostat-s --ip-address --iptables --iso-datetime --iw-scan --jar-manifest --jobs --jwt --kv --last --ls --ls-s --lsblk --lsmod --lsof --lsusb --m3u --mdadm --mount --mpstat --mpstat-s --netstat --nmcli --ntpq --passwd --pidstat --pidstat-s --ping --ping-s --pip-list --pip-show --plist --postconf --ps --route --rpm-qi --rsync --rsync-s --sfdisk --shadow --ss --stat --stat-s --sysctl --syslog-bsd --syslog --systemctl --systemctl-lj --systemctl-ls --systemctl-luf --systeminfo --time --timedatectl --timestamp --top --top-s --tracepath --traceroute --ufw --ufw-appinfo --uname --update-alt-gs --update-alt-q --upower --uptime --url --vmstat --vmstat-s --w --wc --who --x509-cert --xml --xrandr --yaml --zipinfo) jc_parsers_describe=( '--acpi:`acpi` command parser' '--airport:`airport -I` command parser' @@ -103,6 +104,7 @@ _jc() { '--asciitable:ASCII and Unicode table parser' '--asciitable-m:multi-line ASCII and Unicode table parser' '--blkid:`blkid` command parser' + '--cef:CEF string parser' '--chage:`chage --list` command parser' '--cksum:`cksum` and `sum` command parser' '--crontab:`crontab` command and file parser' @@ -153,6 +155,7 @@ _jc() { '--lsof:`lsof` command parser' '--lsusb:`lsusb` command parser' '--m3u:M3U and M3U8 file parser' + '--mdadm:`mdadm` command parser' '--mount:`mount` command parser' '--mpstat:`mpstat` command parser' '--mpstat-s:`mpstat` command streaming parser' @@ -179,6 +182,8 @@ _jc() { '--stat:`stat` command parser' '--stat-s:`stat` command streaming parser' '--sysctl:`sysctl` command parser' + '--syslog-bsd:Syslog RFC 3164 string parser' + '--syslog:Syslog RFC 5424 string parser' '--systemctl:`systemctl` command parser' '--systemctl-lj:`systemctl list-jobs` command parser' '--systemctl-ls:`systemctl list-sockets` command parser' diff --git a/docs/parsers/cef.md b/docs/parsers/cef.md new file mode 100644 index 000000000..1cbec06ca --- /dev/null +++ b/docs/parsers/cef.md @@ -0,0 +1,68 @@ +[Home](https://kellyjonbrazil.github.io/jc/) + + +# jc.parsers.cef + +jc - JSON Convert CEF string parser + +This is a best-effort parser since there are so many variations to CEF +formatting from different vendors. If you require special handling for your +CEF input, you can copy this parser code to the `jc` pluggin directory for +your system and modify it to suit your needs. + +This parser will accept a single CEF string or multiple CEF string lines. +Any text before "CEF" will be ignored. Syslog and CEF escaped characters +(`\\`, `\\"`, `\\]`, `\\|`, `\\n`, `\\r`) are unescaped. To preserve +escaping, use the `--raw` or `raw=True` option in the `parse()` function. + +Usage (cli): + + $ echo 'CEF:0|Vendor|Product|3.2.0|1|SYSTEM|1|... | jc --cef + +Usage (module): + + import jc + result = jc.parse('cef', cef_string_output) + +Schema: + + [ + { + "cef": string, + "bar": boolean, + "baz": integer + } + ] + +Examples: + + $ cef | jc --cef -p + [] + + $ cef | jc --cef -p -r + [] + + + +### parse + +```python +def parse(data: str, raw: bool = False, quiet: bool = False) -> List[Dict] +``` + +Main text parsing function + +Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + +Returns: + + List of Dictionaries. Raw or processed structured data. + +### Parser Information +Compatibility: linux, darwin, cygwin, win32, aix, freebsd + +Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/docs/parsers/mdadm.md b/docs/parsers/mdadm.md new file mode 100644 index 000000000..8522af7c2 --- /dev/null +++ b/docs/parsers/mdadm.md @@ -0,0 +1,64 @@ +[Home](https://kellyjonbrazil.github.io/jc/) + + +# jc.parsers.mdadm + +jc - JSON Convert `mdadm` command output parser + +<> + +Usage (cli): + + $ mdadm | jc --mdadm + + or + + $ jc mdadm + +Usage (module): + + import jc + result = jc.parse('mdadm', mdadm_command_output) + +Schema: + + [ + { + "mdadm": string, + "bar": boolean, + "baz": integer + } + ] + +Examples: + + $ mdadm | jc --mdadm -p + [] + + $ mdadm | jc --mdadm -p -r + [] + + + +### parse + +```python +def parse(data: str, raw: bool = False, quiet: bool = False) -> Dict +``` + +Main text parsing function + +Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + +Returns: + + List of Dictionaries. Raw or processed structured data. + +### Parser Information +Compatibility: linux + +Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/docs/parsers/syslog.md b/docs/parsers/syslog.md new file mode 100644 index 000000000..591c41b13 --- /dev/null +++ b/docs/parsers/syslog.md @@ -0,0 +1,64 @@ +[Home](https://kellyjonbrazil.github.io/jc/) + + +# jc.parsers.syslog + +jc - JSON Convert Syslog RFC 5424 string parser + +<> + +Usage (cli): + + $ syslogstring | jc --syslog + + or + + $ jc syslog-5424 + +Usage (module): + + import jc + result = jc.parse('syslog', syslog_command_output) + +Schema: + + [ + { + "syslog-5424": string, + "bar": boolean, + "baz": integer + } + ] + +Examples: + + $ syslog-5424 | jc --syslog-5424 -p + [] + + $ syslog-5424 | jc --syslog-5424 -p -r + [] + + + +### parse + +```python +def parse(data: str, raw: bool = False, quiet: bool = False) -> List[Dict] +``` + +Main text parsing function + +Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + +Returns: + + List of Dictionaries. Raw or processed structured data. + +### Parser Information +Compatibility: linux, darwin, cygwin, win32, aix, freebsd + +Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/docs/parsers/syslog_bsd.md b/docs/parsers/syslog_bsd.md new file mode 100644 index 000000000..0512b4709 --- /dev/null +++ b/docs/parsers/syslog_bsd.md @@ -0,0 +1,64 @@ +[Home](https://kellyjonbrazil.github.io/jc/) + + +# jc.parsers.syslog\_bsd + +jc - JSON Convert Syslog RFC 3164 string parser + +<> + +Usage (cli): + + $ syslogstring | jc --syslog-bsd + + or + + $ jc syslog-3164 + +Usage (module): + + import jc + result = jc.parse('syslog_bsd', syslog_command_output) + +Schema: + + [ + { + "syslog-3164": string, + "bar": boolean, + "baz": integer + } + ] + +Examples: + + $ syslog-3164 | jc --syslog-3164 -p + [] + + $ syslog-3164 | jc --syslog-3164 -p -r + [] + + + +### parse + +```python +def parse(data: str, raw: bool = False, quiet: bool = False) -> List[Dict] +``` + +Main text parsing function + +Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + +Returns: + + List of Dictionaries. Raw or processed structured data. + +### Parser Information +Compatibility: linux, darwin, cygwin, win32, aix, freebsd + +Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/jc/lib.py b/jc/lib.py index 1c2a07973..0c8f217e5 100644 --- a/jc/lib.py +++ b/jc/lib.py @@ -94,8 +94,8 @@ 'stat', 'stat-s', 'sysctl', - 'syslog-3164', - 'syslog-5424', + 'syslog-bsd', + 'syslog', 'systemctl', 'systemctl-lj', 'systemctl-ls', diff --git a/jc/parsers/syslog_5424.py b/jc/parsers/syslog.py similarity index 55% rename from jc/parsers/syslog_5424.py rename to jc/parsers/syslog.py index 9fa646a22..50b707e97 100644 --- a/jc/parsers/syslog_5424.py +++ b/jc/parsers/syslog.py @@ -4,7 +4,7 @@ Usage (cli): - $ syslog-5424 | jc --syslog-5424 + $ syslogstring | jc --syslog or @@ -13,7 +13,7 @@ Usage (module): import jc - result = jc.parse('syslog_5424', syslog_command_output) + result = jc.parse('syslog', syslog_command_output) Schema: @@ -61,11 +61,45 @@ def _process(proc_data: List[Dict]) -> List[Dict]: List of Dictionaries. Structured to conform to the schema. """ + # fix escape chars specified in syslog RFC 5424 + # https://www.rfc-editor.org/rfc/rfc5424.html#section-6 + escape_map = { + r'\\': '\\', + r'\"': r'"', + r'\]': r']' + } + + structured = re.compile(r''' + (?P\[ + (?P[^\[\=\x22\]\x20]{1,32})\s + (?P[^\[\=\x22\x20]{1,32}=\x22.+\x22\s?)+\] + ) + ''', re.VERBOSE + ) + + each_struct = r'''(?P\[.+?(?[^\[\=\x22\]\x20]{1,32})\s''' + + key_vals = r'''(?P\w+)=(?P\"[^\"]*\")''' + + for item in proc_data: + for key, value in item.copy().items(): + # remove any spaces around values + if item[key]: + item[key] = value.strip() + + # fixup escaped characters + for esc, esc_sub in escape_map.items(): + if item[key]: + item[key] = item[key].replace(esc, esc_sub) + + # parse identity and key value pairs in the structured data section + # if proc_data['structured_data']: + # struct_match = structured.match(proc_data['structured_data']) + # if struct_match: + # struct_dict = struct_match.groupdict() - # process the data here - # rebuild output for added semantic information - # use helper functions in jc.utils for int, float, bool - # conversions and timestamps return proc_data @@ -92,7 +126,7 @@ def parse( jc.utils.input_type_check(data) raw_output: List = [] - syslog_dict = {} + syslog_out = {} # inspired by https://regex101.com/library/Wgbxn2 syslog = re.compile(r''' @@ -120,23 +154,29 @@ def parse( for line in filter(None, data.splitlines()): syslog_match = syslog.match(line) if syslog_match: + syslog_dict = syslog_match.groupdict() + for item in syslog_dict: + if syslog_dict[item] == '-': + syslog_dict[item] = None + priority = None - if syslog_match.group('priority'): - priority = syslog_match.group('priority')[1:-1] - syslog_dict = { + if syslog_dict['priority']: + priority = syslog_dict['priority'][1:-1] + + syslog_out = { 'priority': priority, - 'version': syslog_match.group('version'), - 'timestamp': syslog_match.group('timestamp'), - 'hostname': syslog_match.group('hostname'), - 'appname': syslog_match.group('appname'), - 'proc_id': syslog_match.group('procid'), - 'msg_id': syslog_match.group('msgid'), - 'struct': syslog_match.group('structureddata'), - 'message': syslog_match.group('msg') + 'version': syslog_dict['version'], + 'timestamp': syslog_dict['timestamp'], + 'hostname': syslog_dict['hostname'], + 'appname': syslog_dict['appname'], + 'proc_id': syslog_dict['procid'], + 'msg_id': syslog_dict['msgid'], + 'structured_data': syslog_dict['structureddata'], + 'message': syslog_dict['msg'] } - if syslog_dict: - raw_output.append(syslog_dict) + if syslog_out: + raw_output.append(syslog_out) return raw_output if raw else _process(raw_output) diff --git a/jc/parsers/syslog_3164.py b/jc/parsers/syslog_bsd.py similarity index 96% rename from jc/parsers/syslog_3164.py rename to jc/parsers/syslog_bsd.py index dd6834b23..65b264ed0 100644 --- a/jc/parsers/syslog_3164.py +++ b/jc/parsers/syslog_bsd.py @@ -4,7 +4,7 @@ Usage (cli): - $ syslog-3164 | jc --syslog-3164 + $ syslogstring | jc --syslog-bsd or @@ -13,7 +13,7 @@ Usage (module): import jc - result = jc.parse('syslog_3164', syslog_command_output) + result = jc.parse('syslog_bsd', syslog_command_output) Schema: diff --git a/man/jc.1 b/man/jc.1 index b237c3d8e..312dc9e0e 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -1,4 +1,4 @@ -.TH jc 1 2022-08-06 1.20.5 "JSON Convert" +.TH jc 1 2022-08-12 1.20.5 "JSON Convert" .SH NAME \fBjc\fP \- JSON Convert JSONifies the output of many CLI tools and file-types .SH SYNOPSIS @@ -52,6 +52,11 @@ multi-line ASCII and Unicode table parser \fB--blkid\fP `blkid` command parser +.TP +.B +\fB--cef\fP +CEF string parser + .TP .B \fB--chage\fP @@ -302,6 +307,11 @@ Key/Value file parser \fB--m3u\fP M3U and M3U8 file parser +.TP +.B +\fB--mdadm\fP +`mdadm` command parser + .TP .B \fB--mount\fP @@ -432,6 +442,16 @@ PLIST file parser \fB--sysctl\fP `sysctl` command parser +.TP +.B +\fB--syslog-bsd\fP +Syslog RFC 3164 string parser + +.TP +.B +\fB--syslog\fP +Syslog RFC 5424 string parser + .TP .B \fB--systemctl\fP From c24c5f7985f41c38b7bfdce7ea0c964b6a75dc17 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 12 Aug 2022 17:35:28 -0700 Subject: [PATCH 057/116] extract structured data. need to fix corner cases --- jc/parsers/syslog.py | 77 +++++++++++++++++++++++++++++++------------- 1 file changed, 55 insertions(+), 22 deletions(-) diff --git a/jc/parsers/syslog.py b/jc/parsers/syslog.py index 50b707e97..722f45501 100644 --- a/jc/parsers/syslog.py +++ b/jc/parsers/syslog.py @@ -34,7 +34,7 @@ [] """ import re -from typing import List, Dict +from typing import List, Dict, Optional import jc.utils @@ -49,6 +49,39 @@ class info(): __version__ = info.version +def _extract_structs(structs_string: str) -> List[str]: + struct_match = re.compile(r'(?P\[.+?(? Optional[str]: + ident = re.compile(r'''\[(?P[^\[\=\x22\]\x20]{1,32})\s''') + ident_match = ident.search(struct_string) + if ident_match: + return ident_match.group('ident') + return None + + +def _extract_kv(struct_string) -> List[Dict]: + key_vals = re.compile(r'''(?P\w+)=(?P\"[^\"]*\")''') + key_vals_match = key_vals.findall(struct_string) + kv_list = [] + + if key_vals_match: + for kv in key_vals_match: + key, val = kv + kv_list.append({key: val[1:-1]}) + + return kv_list + + def _process(proc_data: List[Dict]) -> List[Dict]: """ Final processing to conform to the schema. @@ -69,37 +102,37 @@ def _process(proc_data: List[Dict]) -> List[Dict]: r'\]': r']' } - structured = re.compile(r''' - (?P\[ - (?P[^\[\=\x22\]\x20]{1,32})\s - (?P[^\[\=\x22\x20]{1,32}=\x22.+\x22\s?)+\] - ) - ''', re.VERBOSE - ) - - each_struct = r'''(?P\[.+?(?[^\[\=\x22\]\x20]{1,32})\s''' - - key_vals = r'''(?P\w+)=(?P\"[^\"]*\")''' - for item in proc_data: - for key, value in item.copy().items(): + for key, value in item.items(): # remove any spaces around values if item[key]: item[key] = value.strip() # fixup escaped characters for esc, esc_sub in escape_map.items(): - if item[key]: + if item[key] and isinstance(item[key], str): item[key] = item[key].replace(esc, esc_sub) - # parse identity and key value pairs in the structured data section - # if proc_data['structured_data']: - # struct_match = structured.match(proc_data['structured_data']) - # if struct_match: - # struct_dict = struct_match.groupdict() + # parse identity and key value pairs in the structured data section + structs = None + if item['structured_data']: + structs_list = [] + structs = _extract_structs(item['structured_data']) + + for a_struct in structs: + struct_obj = { + 'identity': _extract_ident(a_struct) + } + + my_values = {} + + for val_obj in _extract_kv(a_struct): + my_values.update(val_obj) + + struct_obj.update({'values': my_values}) # type: ignore + structs_list.append(struct_obj) + item['structured_data'] = structs_list return proc_data From 9adc66af63c6648f32381b3610206e1915f68cb6 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 12 Aug 2022 17:35:47 -0700 Subject: [PATCH 058/116] version bump to 1.21.0 --- CHANGELOG | 2 +- jc/lib.py | 2 +- man/jc.1 | 2 +- setup.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index f60202468..523c1462c 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,6 +1,6 @@ jc changelog -xxxxxxxx v1.20.5 +xxxxxxxx v1.21.0 - Add IP Address string parser - Add Syslog string parsers (RFC 3164 and RFC 5424) - Add CEF string parser diff --git a/jc/lib.py b/jc/lib.py index 0c8f217e5..e88613cfa 100644 --- a/jc/lib.py +++ b/jc/lib.py @@ -6,7 +6,7 @@ from typing import Dict, List, Iterable, Union, Iterator from jc import appdirs -__version__ = '1.20.5' +__version__ = '1.21.0' parsers = [ 'acpi', diff --git a/man/jc.1 b/man/jc.1 index 312dc9e0e..9d0984ca4 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -1,4 +1,4 @@ -.TH jc 1 2022-08-12 1.20.5 "JSON Convert" +.TH jc 1 2022-08-12 1.21.0 "JSON Convert" .SH NAME \fBjc\fP \- JSON Convert JSONifies the output of many CLI tools and file-types .SH SYNOPSIS diff --git a/setup.py b/setup.py index 5f39bc3cb..d92354ab6 100755 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setuptools.setup( name='jc', - version='1.20.5', + version='1.21.0', author='Kelly Brazil', author_email='kellyjonbrazil@gmail.com', description='Converts the output of popular command-line tools and file-types to JSON.', From 2d4ace68e76f6735ab6a655373778f77cba00328 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 12 Aug 2022 18:04:22 -0700 Subject: [PATCH 059/116] fix escape character fixups --- jc/parsers/syslog.py | 51 ++++++++++++++++++++++++++++++-------------- 1 file changed, 35 insertions(+), 16 deletions(-) diff --git a/jc/parsers/syslog.py b/jc/parsers/syslog.py index 722f45501..552cd07d7 100644 --- a/jc/parsers/syslog.py +++ b/jc/parsers/syslog.py @@ -19,9 +19,22 @@ [ { - "syslog-5424": string, - "bar": boolean, - "baz": integer + "priority": integer, + "version": integer, + "timestamp": string, # add epoch fields + "hostname": string, + "appname": string, + "proc_id": integer, + "msg_id": string, + "structured_data": [ + { + "identity": string, + "values": { + "": string + } + } + ], + "message": string } ] @@ -49,6 +62,15 @@ class info(): __version__ = info.version +# fix escape chars specified in syslog RFC 5424 +# https://www.rfc-editor.org/rfc/rfc5424.html#section-6 +escape_map = { + r'\\': '\\', + r'\"': r'"', + r'\]': r']' +} + + def _extract_structs(structs_string: str) -> List[str]: struct_match = re.compile(r'(?P\[.+?(? List[str]: def _extract_ident(struct_string) -> Optional[str]: - ident = re.compile(r'''\[(?P[^\[\=\x22\]\x20]{1,32})\s''') + ident = re.compile(r'\[(?P[^\[\=\x22\]\x20]{1,32})\s') ident_match = ident.search(struct_string) if ident_match: return ident_match.group('ident') @@ -70,13 +92,18 @@ def _extract_ident(struct_string) -> Optional[str]: def _extract_kv(struct_string) -> List[Dict]: - key_vals = re.compile(r'''(?P\w+)=(?P\"[^\"]*\")''') + key_vals = re.compile(r'(?P\w+)=(?P\"[^\"]*\")') key_vals_match = key_vals.findall(struct_string) kv_list = [] if key_vals_match: for kv in key_vals_match: key, val = kv + + # fixup escaped characters + for esc, esc_sub in escape_map.items(): + val = val.replace(esc, esc_sub) + kv_list.append({key: val[1:-1]}) return kv_list @@ -94,24 +121,16 @@ def _process(proc_data: List[Dict]) -> List[Dict]: List of Dictionaries. Structured to conform to the schema. """ - # fix escape chars specified in syslog RFC 5424 - # https://www.rfc-editor.org/rfc/rfc5424.html#section-6 - escape_map = { - r'\\': '\\', - r'\"': r'"', - r'\]': r']' - } - for item in proc_data: for key, value in item.items(): # remove any spaces around values if item[key]: item[key] = value.strip() - # fixup escaped characters + # fixup escaped characters + if item['message']: for esc, esc_sub in escape_map.items(): - if item[key] and isinstance(item[key], str): - item[key] = item[key].replace(esc, esc_sub) + item['message'] = item['message'].replace(esc, esc_sub) # parse identity and key value pairs in the structured data section structs = None From 850a02b4c0ef86922af7c966f88e11893ec2fe81 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 12 Aug 2022 18:20:11 -0700 Subject: [PATCH 060/116] formatting --- jc/parsers/syslog.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/jc/parsers/syslog.py b/jc/parsers/syslog.py index 552cd07d7..531dca616 100644 --- a/jc/parsers/syslog.py +++ b/jc/parsers/syslog.py @@ -38,6 +38,8 @@ } ] + Blank values will be null/None + Examples: $ syslog-5424 | jc --syslog-5424 -p @@ -133,7 +135,6 @@ def _process(proc_data: List[Dict]) -> List[Dict]: item['message'] = item['message'].replace(esc, esc_sub) # parse identity and key value pairs in the structured data section - structs = None if item['structured_data']: structs_list = [] structs = _extract_structs(item['structured_data']) @@ -184,7 +185,8 @@ def parse( syslog = re.compile(r''' (?P<(\d|\d{2}|1[1-8]\d|19[01])>)? (?P\d{1,2})?\s* - (?P-|(?P[12]\d{3})- + (?P-| + (?P[12]\d{3})- (?P0\d|[1][012])- (?P[012]\d|3[01])T (?P[01]\d|2[0-4]): From 4d3d608e5cf40c7ba0fd794157213803b98d3867 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sun, 14 Aug 2022 10:52:58 -0700 Subject: [PATCH 061/116] add support for ISO-like formats --- jc/utils.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/jc/utils.py b/jc/utils.py index f7f195b1f..6f964476b 100644 --- a/jc/utils.py +++ b/jc/utils.py @@ -393,6 +393,9 @@ def _parse_dt(dt_string, format_hint=None): # sometimes UTC is referenced as 'Coordinated Universal Time'. Convert to 'UTC' data = data.replace('Coordinated Universal Time', 'UTC') + # UTC can also be indicated with 'Z' for Zulu time (ISO-8601). Convert to 'UTC' + data = data.replace('Z', 'UTC') + if 'UTC' in data: utc_tz = True if 'UTC+' in data or 'UTC-' in data: @@ -404,6 +407,8 @@ def _parse_dt(dt_string, format_hint=None): formats = [ {'id': 1000, 'format': '%a %b %d %H:%M:%S %Y', 'locale': None}, # manual C locale format conversion: Tue Mar 23 16:12:11 2021 or Tue Mar 23 16:12:11 IST 2021 {'id': 1100, 'format': '%a %b %d %H:%M:%S %Y %z', 'locale': None}, # git date output: Thu Mar 5 09:17:40 2020 -0800 + {'id': 1300, 'format': '%Y-%m-%dT%H:%M:%S.%f%Z', 'locale': None}, # ISO Format with UTC (found in syslog 5424): 2003-10-11T22:14:15.003Z + {'id': 1310, 'format': '%Y-%m-%dT%H:%M:%S.%f', 'locale': None}, # ISO Format without UTC (found in syslog 5424): 2003-10-11T22:14:15.003 {'id': 1500, 'format': '%Y-%m-%d %H:%M', 'locale': None}, # en_US.UTF-8 local format (found in who cli output): 2021-03-23 00:14 {'id': 1600, 'format': '%m/%d/%Y %I:%M %p', 'locale': None}, # Windows english format (found in dir cli output): 12/07/2019 02:09 AM {'id': 1700, 'format': '%m/%d/%Y, %I:%M:%S %p', 'locale': None}, # Windows english format wint non-UTC tz (found in systeminfo cli output): 3/22/2021, 1:15:51 PM (UTC-0600) @@ -450,7 +455,7 @@ def _parse_dt(dt_string, format_hint=None): 'TOT', 'TRT', 'TVT', 'U', 'ULAST', 'ULAT', 'UYST', 'UYT', 'UZT', 'V', 'VET', 'VLAST', 'VLAT', 'VOST', 'VUT', 'W', 'WAKT', 'WARST', 'WAST', 'WAT', 'WEST', 'WET', 'WFT', 'WGST', 'WGT', 'WIB', 'WIT', 'WITA', 'WST', 'WT', 'X', 'Y', 'YAKST', 'YAKT', 'YAPT', - 'YEKST', 'YEKT', 'Z', 'UTC-1200', 'UTC-1100', 'UTC-1000', 'UTC-0930', 'UTC-0900', + 'YEKST', 'YEKT', 'UTC-1200', 'UTC-1100', 'UTC-1000', 'UTC-0930', 'UTC-0900', 'UTC-0800', 'UTC-0700', 'UTC-0600', 'UTC-0500', 'UTC-0400', 'UTC-0300', 'UTC-0230', 'UTC-0200', 'UTC-0100', 'UTC+0100', 'UTC+0200', 'UTC+0300', 'UTC+0400', 'UTC+0430', 'UTC+0500', 'UTC+0530', 'UTC+0545', 'UTC+0600', 'UTC+0630', 'UTC+0700', 'UTC+0800', From 23263bd83e9a5d21b9c2093e750a85a058e2fa5e Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sun, 14 Aug 2022 11:48:24 -0700 Subject: [PATCH 062/116] add support for strings with UTC offset suffixes --- jc/utils.py | 93 +++++++++++++++++++++++++++++++---------------------- 1 file changed, 54 insertions(+), 39 deletions(-) diff --git a/jc/utils.py b/jc/utils.py index 6f964476b..ebdc5ae01 100644 --- a/jc/utils.py +++ b/jc/utils.py @@ -370,45 +370,11 @@ def _parse_dt(dt_string, format_hint=None): If the conversion completely fails, all fields will be None. """ - data = dt_string or '' - normalized_datetime = '' - utc_tz = False - dt = None - dt_utc = None - timestamp_naive = None - timestamp_utc = None - timestamp_obj = { - 'format': None, - 'timestamp_naive': None, - 'timestamp_utc': None - } - utc_tz = False - - # convert format_hint to a tuple so it is hashable (for lru_cache) - if not format_hint: - format_hint = tuple() - else: - format_hint = tuple(format_hint) - - # sometimes UTC is referenced as 'Coordinated Universal Time'. Convert to 'UTC' - data = data.replace('Coordinated Universal Time', 'UTC') - - # UTC can also be indicated with 'Z' for Zulu time (ISO-8601). Convert to 'UTC' - data = data.replace('Z', 'UTC') - - if 'UTC' in data: - utc_tz = True - if 'UTC+' in data or 'UTC-' in data: - utc_tz = bool('UTC+0000' in data or 'UTC-0000' in data) - - elif '+0000' in data or '-0000' in data: - utc_tz = True - - formats = [ + formats = ( {'id': 1000, 'format': '%a %b %d %H:%M:%S %Y', 'locale': None}, # manual C locale format conversion: Tue Mar 23 16:12:11 2021 or Tue Mar 23 16:12:11 IST 2021 {'id': 1100, 'format': '%a %b %d %H:%M:%S %Y %z', 'locale': None}, # git date output: Thu Mar 5 09:17:40 2020 -0800 {'id': 1300, 'format': '%Y-%m-%dT%H:%M:%S.%f%Z', 'locale': None}, # ISO Format with UTC (found in syslog 5424): 2003-10-11T22:14:15.003Z - {'id': 1310, 'format': '%Y-%m-%dT%H:%M:%S.%f', 'locale': None}, # ISO Format without UTC (found in syslog 5424): 2003-10-11T22:14:15.003 + {'id': 1310, 'format': '%Y-%m-%dT%H:%M:%S.%f', 'locale': None}, # ISO Format without TZ (found in syslog 5424): 2003-10-11T22:14:15.003 {'id': 1500, 'format': '%Y-%m-%d %H:%M', 'locale': None}, # en_US.UTF-8 local format (found in who cli output): 2021-03-23 00:14 {'id': 1600, 'format': '%m/%d/%Y %I:%M %p', 'locale': None}, # Windows english format (found in dir cli output): 12/07/2019 02:09 AM {'id': 1700, 'format': '%m/%d/%Y, %I:%M:%S %p', 'locale': None}, # Windows english format wint non-UTC tz (found in systeminfo cli output): 3/22/2021, 1:15:51 PM (UTC-0600) @@ -431,11 +397,11 @@ def _parse_dt(dt_string, format_hint=None): {'id': 8200, 'format': '%A %d %B %Y, %H:%M:%S UTC%z', 'locale': ''}, # fr_FR.utf8 locale format (found in date cli output): vendredi 26 mars 2021, 13:26:46 (UTC+0000) {'id': 8300, 'format': '%A %d %B %Y, %H:%M:%S', 'locale': ''}, # fr_FR.utf8 locale format with non-UTC tz (found in date cli output): vendredi 26 mars 2021, 13:26:46 (UTC-0400) {'id': 9000, 'format': '%c', 'locale': ''} # locally configured locale format conversion: Could be anything :) this is a last-gasp attempt - ] + ) # from https://www.timeanddate.com/time/zones/ # only removed UTC timezone and added known non-UTC offsets - tz_abbr = [ + tz_abbr = { 'A', 'ACDT', 'ACST', 'ACT', 'ACWST', 'ADT', 'AEDT', 'AEST', 'AET', 'AFT', 'AKDT', 'AKST', 'ALMT', 'AMST', 'AMT', 'ANAST', 'ANAT', 'AQTT', 'ART', 'AST', 'AT', 'AWDT', 'AWST', 'AZOST', 'AZOT', 'AZST', 'AZT', 'AoE', 'B', 'BNT', 'BOT', 'BRST', 'BRT', 'BST', @@ -461,7 +427,50 @@ def _parse_dt(dt_string, format_hint=None): 'UTC+0500', 'UTC+0530', 'UTC+0545', 'UTC+0600', 'UTC+0630', 'UTC+0700', 'UTC+0800', 'UTC+0845', 'UTC+0900', 'UTC+1000', 'UTC+1030', 'UTC+1100', 'UTC+1200', 'UTC+1300', 'UTC+1345', 'UTC+1400' - ] + } + + offset_suffixes = ( + '-12:00', '-11:00', '-10:00', '-09:30', '-09:00', + '-08:00', '-07:00', '-06:00', '-05:00', '-04:00', '-03:00', '-02:30', + '-02:00', '-01:00', '+01:00', '+02:00', '+03:00', '+04:00', '+04:30', + '+05:00', '+05:30', '+05:45', '+06:00', '+06:30', '+07:00', '+08:00', + '+08:45', '+09:00', '+10:00', '+10:30', '+11:00', '+12:00', '+13:00', + '+13:45', '+14:00' + ) + + data = dt_string or '' + normalized_datetime = '' + utc_tz = False + dt = None + dt_utc = None + timestamp_naive = None + timestamp_utc = None + timestamp_obj = { + 'format': None, + 'timestamp_naive': None, + 'timestamp_utc': None + } + utc_tz = False + + # convert format_hint to a tuple so it is hashable (for lru_cache) + if not format_hint: + format_hint = tuple() + else: + format_hint = tuple(format_hint) + + # sometimes UTC is referenced as 'Coordinated Universal Time'. Convert to 'UTC' + data = data.replace('Coordinated Universal Time', 'UTC') + + # UTC can also be indicated with 'Z' for Zulu time (ISO-8601). Convert to 'UTC' + data = data.replace('Z', 'UTC') + + if 'UTC' in data: + utc_tz = True + if 'UTC+' in data or 'UTC-' in data: + utc_tz = bool('UTC+0000' in data or 'UTC-0000' in data) + + elif '+0000' in data or '-0000' in data: + utc_tz = True # normalize the timezone by taking out any timezone reference, except UTC cleandata = data.replace('(', '').replace(')', '') @@ -472,6 +481,12 @@ def _parse_dt(dt_string, format_hint=None): normalized_datetime = ' '.join(normalized_datetime_list) + # remove non UTC offset suffixes at the end of the string + for suffix in offset_suffixes: + if normalized_datetime.endswith(suffix): + normalized_datetime = normalized_datetime[0:-len(suffix)] + break + # normalize further by converting any greater-than 6-digit subsecond to 6-digits p = re.compile(r'(\W\d\d:\d\d:\d\d\.\d{6})\d+\W') normalized_datetime = p.sub(r'\g<1> ', normalized_datetime) From 81ed9c0c5cea3d6df60d593e1421ac998c9b1453 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sun, 14 Aug 2022 11:48:41 -0700 Subject: [PATCH 063/116] add timestamps --- jc/parsers/syslog.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/jc/parsers/syslog.py b/jc/parsers/syslog.py index 531dca616..ad38e494b 100644 --- a/jc/parsers/syslog.py +++ b/jc/parsers/syslog.py @@ -129,6 +129,13 @@ def _process(proc_data: List[Dict]) -> List[Dict]: if item[key]: item[key] = value.strip() + # add timestamp fields + if item['timestamp']: + format = (1300, 1310) + dt = jc.utils.timestamp(item['timestamp'], format) + item['timestamp_epoch'] = dt.naive + item['timestamp_epoch_utc'] = dt.utc + # fixup escaped characters if item['message']: for esc, esc_sub in escape_map.items(): From 20982ab1a6a12b68cd36d2c1b6a720ecb590341d Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sun, 14 Aug 2022 12:03:16 -0700 Subject: [PATCH 064/116] doc update --- docs/parsers/syslog.md | 80 +++++++++++++++++++++++++++++++++++++----- jc/parsers/syslog.py | 70 ++++++++++++++++++++++++++++++++---- man/jc.1 | 2 +- 3 files changed, 136 insertions(+), 16 deletions(-) diff --git a/docs/parsers/syslog.md b/docs/parsers/syslog.md index 591c41b13..1e9628dd8 100644 --- a/docs/parsers/syslog.md +++ b/docs/parsers/syslog.md @@ -5,7 +5,11 @@ jc - JSON Convert Syslog RFC 5424 string parser -<> +The `timestamp_epoch` calculated timestamp field is naive. (i.e. based on +the local time of the system the parser is run on) + +The `timestamp_epoch_utc` calculated timestamp field is timezone-aware and +is only available if the timezone field is UTC. Usage (cli): @@ -21,22 +25,82 @@ Usage (module): result = jc.parse('syslog', syslog_command_output) Schema: +Blank values converted to `null`/`None` [ { - "syslog-5424": string, - "bar": boolean, - "baz": integer + "priority": integer, + "version": integer, + "timestamp": string, + "timestamp_epoch": integer, # [0] + "timestamp_epoch_utc": integer, # [1] + "hostname": string, + "appname": string, + "proc_id": integer, + "msg_id": string, + "structured_data": [ + { + "identity": string, + "values": { + "": string + } + } + ], + "message": string } ] + [0] naive timestamp if "timestamp" field is parsable, else null + [1] timezone aware timestamp availabe for UTC, else null + Examples: - $ syslog-5424 | jc --syslog-5424 -p - [] + $ cat syslog.txt| jc --syslog -p + [ + { + "priority": 35, + "version": 1, + "timestamp": "2003-10-11T22:14:15.003Z", + "hostname": "mymachine.example.com", + "appname": "evntslog", + "proc_id": null, + "msg_id": "ID47", + "structured_data": [ + { + "identity": "exampleSDID@32473", + "values": { + "iut": "3", + "eventSource": "Application", + "eventID": "1011" + } + }, + { + "identity": "examplePriority@32473", + "values": { + "class": "high" + } + } + ], + "message": "unauthorized attempt", + "timestamp_epoch": 1065935655, + "timestamp_epoch_utc": 1065910455 + } + ] - $ syslog-5424 | jc --syslog-5424 -p -r - [] + $ cat syslog.txt| jc --syslog -p -r + [ + { + "priority": "35", + "version": "1", + "timestamp": "2003-10-11T22:14:15.003Z", + "hostname": "mymachine.example.com", + "appname": "evntslog", + "proc_id": null, + "msg_id": "ID47", + "structured_data": "[exampleSDID@32473 iut=\"3\" eventSource=\...", + "message": "unauthorized attempt" + } + ] diff --git a/jc/parsers/syslog.py b/jc/parsers/syslog.py index ad38e494b..97b68cc22 100644 --- a/jc/parsers/syslog.py +++ b/jc/parsers/syslog.py @@ -1,6 +1,10 @@ """jc - JSON Convert Syslog RFC 5424 string parser -<> +The `timestamp_epoch` calculated timestamp field is naive. (i.e. based on +the local time of the system the parser is run on) + +The `timestamp_epoch_utc` calculated timestamp field is timezone-aware and +is only available if the timezone field is UTC. Usage (cli): @@ -16,12 +20,15 @@ result = jc.parse('syslog', syslog_command_output) Schema: +Blank values converted to `null`/`None` [ { "priority": integer, "version": integer, - "timestamp": string, # add epoch fields + "timestamp": string, + "timestamp_epoch": integer, # [0] + "timestamp_epoch_utc": integer, # [1] "hostname": string, "appname": string, "proc_id": integer, @@ -38,15 +45,57 @@ } ] - Blank values will be null/None + [0] naive timestamp if "timestamp" field is parsable, else null + [1] timezone aware timestamp availabe for UTC, else null Examples: - $ syslog-5424 | jc --syslog-5424 -p - [] + $ cat syslog.txt| jc --syslog -p + [ + { + "priority": 35, + "version": 1, + "timestamp": "2003-10-11T22:14:15.003Z", + "hostname": "mymachine.example.com", + "appname": "evntslog", + "proc_id": null, + "msg_id": "ID47", + "structured_data": [ + { + "identity": "exampleSDID@32473", + "values": { + "iut": "3", + "eventSource": "Application", + "eventID": "1011" + } + }, + { + "identity": "examplePriority@32473", + "values": { + "class": "high" + } + } + ], + "message": "unauthorized attempt", + "timestamp_epoch": 1065935655, + "timestamp_epoch_utc": 1065910455 + } + ] - $ syslog-5424 | jc --syslog-5424 -p -r - [] + $ cat syslog.txt| jc --syslog -p -r + [ + { + "priority": "35", + "version": "1", + "timestamp": "2003-10-11T22:14:15.003Z", + "hostname": "mymachine.example.com", + "appname": "evntslog", + "proc_id": null, + "msg_id": "ID47", + "structured_data": "[exampleSDID@32473 iut=\"3\" eventSource=\...", + "message": "unauthorized attempt" + } + ] """ import re from typing import List, Dict, Optional @@ -123,6 +172,8 @@ def _process(proc_data: List[Dict]) -> List[Dict]: List of Dictionaries. Structured to conform to the schema. """ + int_list = {'priority', 'version', 'proc_id'} + for item in proc_data: for key, value in item.items(): # remove any spaces around values @@ -161,6 +212,11 @@ def _process(proc_data: List[Dict]) -> List[Dict]: item['structured_data'] = structs_list + # integer conversions + for key in item: + if key in int_list: + item[key] = jc.utils.convert_to_int(item[key]) + return proc_data diff --git a/man/jc.1 b/man/jc.1 index 9d0984ca4..7029fe7fe 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -1,4 +1,4 @@ -.TH jc 1 2022-08-12 1.21.0 "JSON Convert" +.TH jc 1 2022-08-14 1.21.0 "JSON Convert" .SH NAME \fBjc\fP \- JSON Convert JSONifies the output of many CLI tools and file-types .SH SYNOPSIS From 0cd471dfb22f9be12c9c7207faacde21479b1f58 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sun, 14 Aug 2022 12:11:31 -0700 Subject: [PATCH 065/116] doc update --- docs/parsers/syslog.md | 13 ++++++------- jc/parsers/syslog.py | 13 ++++++------- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/docs/parsers/syslog.md b/docs/parsers/syslog.md index 1e9628dd8..e03d791f9 100644 --- a/docs/parsers/syslog.md +++ b/docs/parsers/syslog.md @@ -5,6 +5,9 @@ jc - JSON Convert Syslog RFC 5424 string parser +This parser accepts a single syslog line string or multiple syslog lines +separated by newlines. + The `timestamp_epoch` calculated timestamp field is naive. (i.e. based on the local time of the system the parser is run on) @@ -13,16 +16,12 @@ is only available if the timezone field is UTC. Usage (cli): - $ syslogstring | jc --syslog - - or - - $ jc syslog-5424 + $ echo <165>1 2003-08-24T05:14:15.000003-07:00 192.0.2... | jc --syslog Usage (module): import jc - result = jc.parse('syslog', syslog_command_output) + result = jc.parse('syslog', syslog_string) Schema: Blank values converted to `null`/`None` @@ -97,7 +96,7 @@ Examples: "appname": "evntslog", "proc_id": null, "msg_id": "ID47", - "structured_data": "[exampleSDID@32473 iut=\"3\" eventSource=\...", + "structured_data": "[exampleSDID@32473 iut=\\"3\\" eventSource...", "message": "unauthorized attempt" } ] diff --git a/jc/parsers/syslog.py b/jc/parsers/syslog.py index 97b68cc22..4335cae70 100644 --- a/jc/parsers/syslog.py +++ b/jc/parsers/syslog.py @@ -1,5 +1,8 @@ """jc - JSON Convert Syslog RFC 5424 string parser +This parser accepts a single syslog line string or multiple syslog lines +separated by newlines. + The `timestamp_epoch` calculated timestamp field is naive. (i.e. based on the local time of the system the parser is run on) @@ -8,16 +11,12 @@ Usage (cli): - $ syslogstring | jc --syslog - - or - - $ jc syslog-5424 + $ echo <165>1 2003-08-24T05:14:15.000003-07:00 192.0.2... | jc --syslog Usage (module): import jc - result = jc.parse('syslog', syslog_command_output) + result = jc.parse('syslog', syslog_string) Schema: Blank values converted to `null`/`None` @@ -92,7 +91,7 @@ "appname": "evntslog", "proc_id": null, "msg_id": "ID47", - "structured_data": "[exampleSDID@32473 iut=\"3\" eventSource=\...", + "structured_data": "[exampleSDID@32473 iut=\\"3\\" eventSource...", "message": "unauthorized attempt" } ] From 2fda7af8a74efb837ade7ea535557317d299dea9 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sun, 14 Aug 2022 12:18:22 -0700 Subject: [PATCH 066/116] change values to parameters in schema --- docs/parsers/syslog.md | 6 +++--- jc/parsers/syslog.py | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/parsers/syslog.md b/docs/parsers/syslog.md index e03d791f9..9ce7f353b 100644 --- a/docs/parsers/syslog.md +++ b/docs/parsers/syslog.md @@ -40,7 +40,7 @@ Blank values converted to `null`/`None` "structured_data": [ { "identity": string, - "values": { + "parameters": { "": string } } @@ -67,7 +67,7 @@ Examples: "structured_data": [ { "identity": "exampleSDID@32473", - "values": { + "parameters": { "iut": "3", "eventSource": "Application", "eventID": "1011" @@ -75,7 +75,7 @@ Examples: }, { "identity": "examplePriority@32473", - "values": { + "parameters": { "class": "high" } } diff --git a/jc/parsers/syslog.py b/jc/parsers/syslog.py index 4335cae70..938ce37d1 100644 --- a/jc/parsers/syslog.py +++ b/jc/parsers/syslog.py @@ -35,7 +35,7 @@ "structured_data": [ { "identity": string, - "values": { + "parameters": { "": string } } @@ -62,7 +62,7 @@ "structured_data": [ { "identity": "exampleSDID@32473", - "values": { + "parameters": { "iut": "3", "eventSource": "Application", "eventID": "1011" @@ -70,7 +70,7 @@ }, { "identity": "examplePriority@32473", - "values": { + "parameters": { "class": "high" } } @@ -206,7 +206,7 @@ def _process(proc_data: List[Dict]) -> List[Dict]: for val_obj in _extract_kv(a_struct): my_values.update(val_obj) - struct_obj.update({'values': my_values}) # type: ignore + struct_obj.update({'parameters': my_values}) # type: ignore structs_list.append(struct_obj) item['structured_data'] = structs_list From 0bc7311473ae9cf7ae0fa5947b9fd119d17ec318 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Mon, 15 Aug 2022 11:14:17 -0700 Subject: [PATCH 067/116] doc update --- docs/parsers/mdadm.md | 202 +++++++++++++++++++++++++++++++++++++---- jc/parsers/mdadm.py | 206 ++++++++++++++++++++++++++++++++++++++---- man/jc.1 | 2 +- 3 files changed, 373 insertions(+), 37 deletions(-) diff --git a/docs/parsers/mdadm.md b/docs/parsers/mdadm.md index 8522af7c2..f0cd7d221 100644 --- a/docs/parsers/mdadm.md +++ b/docs/parsers/mdadm.md @@ -5,15 +5,23 @@ jc - JSON Convert `mdadm` command output parser -<> +Supports the `--query` and `--examine` options in `mdadm`. Usage (cli): - $ mdadm | jc --mdadm + $ mdadm --query --detail /dev/md0 | jc --mdadm - or +or - $ jc mdadm + $ mdadm --examine -E /dev/sdb1 | jc --mdadm + +or + + $ jc mdadm --query --detail /dev/md0 + +or + + $ jc mdadm --examine -E /dev/sdb1 Usage (module): @@ -22,21 +30,181 @@ Usage (module): Schema: - [ - { - "mdadm": string, - "bar": boolean, - "baz": integer - } - ] + { + "device": string, + "magic": string, + "version": string, + "feature_map": string, + "array_uuid": string, + "name": string, + "name_val": string, + "uuid": string, + "uuid_val": string, + "creation_time": string, + "creation_time_epoch": integer, # naive timestamp + "raid_level": string, + "array_size": string, + "array_size_num": integer, + "used_dev_size": string, + "used_dev_size_num": integer, + "raid_devices": integer, + "avail_dev_size": string, + "avail_dev_size_num": integer, + "data_offset": integer, + "super_offset": integer, + "unused_space": string, + "unused_space_before": integer, + "unused_space_after": integer, + "state": string, + "state_list": [ + string + ], + "device_uuid": string, + "flags": string, + "flag_list": [ + string + ], + "update_time": string, + "update_time_epoch": integer, # naive timestamp + "bad_block_log": string, + "checksum": string, + "checksum_val": string, + "checksum_state": string, + "events": integer, + "chunk_size": integer, + "device_role": string, + "array_state": string, + "array_state_list": [ + string + ], + "consistency_policy": string, + "rebuild_status": string, + "rebuild_status_percent": integer, + "resync_status": string, + "resync_status_percent": integer, + "check_status": string, + "check_status_percent": integer, + "total_devices": integer, + "preferred_minor": integer, + "persistence": string, + "active_devices": integer, + "working_devices": integer, + "failed_devices": integer, + "spare_devices": integer, + "device_table": [ + { + "number": integer/null, + "major": integer/null, + "minor": integer/null, + "state": [ + string + ], + "device": string, + "raid_device": integer/null + } + ] + } Examples: - $ mdadm | jc --mdadm -p - [] - - $ mdadm | jc --mdadm -p -r - [] + $ mdadm --query --detail /dev/md0 | jc --mdadm -p + { + "device": "/dev/md0", + "version": "1.1", + "creation_time": "Tue Apr 13 23:22:16 2010", + "raid_level": "raid1", + "array_size": "5860520828 (5.46 TiB 6.00 TB)", + "used_dev_size": "5860520828 (5.46 TiB 6.00 TB)", + "raid_devices": 2, + "total_devices": 2, + "persistence": "Superblock is persistent", + "intent_bitmap": "Internal", + "update_time": "Tue Jul 26 20:16:31 2022", + "state": "clean", + "active_devices": 2, + "working_devices": 2, + "failed_devices": 0, + "spare_devices": 0, + "consistency_policy": "bitmap", + "name": "virttest:0", + "uuid": "85c5b164:d58a5ada:14f5fe07:d642e843", + "events": 2193679, + "device_table": [ + { + "number": 3, + "major": 8, + "minor": 17, + "state": [ + "active", + "sync" + ], + "device": "/dev/sdb1", + "raid_device": 0 + }, + { + "number": 2, + "major": 8, + "minor": 33, + "state": [ + "active", + "sync" + ], + "device": "/dev/sdc1", + "raid_device": 1 + } + ], + "array_size_num": 5860520828, + "used_dev_size_num": 5860520828, + "name_val": "virttest:0", + "uuid_val": "85c5b164:d58a5ada:14f5fe07:d642e843", + "state_list": [ + "clean" + ], + "creation_time_epoch": 1271226136, + "update_time_epoch": 1658891791 + } + + $ mdadm --query --detail /dev/md0 | jc --mdadm -p -r + { + "device": "/dev/md0", + "version": "1.1", + "creation_time": "Tue Apr 13 23:22:16 2010", + "raid_level": "raid1", + "array_size": "5860520828 (5.46 TiB 6.00 TB)", + "used_dev_size": "5860520828 (5.46 TiB 6.00 TB)", + "raid_devices": "2", + "total_devices": "2", + "persistence": "Superblock is persistent", + "intent_bitmap": "Internal", + "update_time": "Tue Jul 26 20:16:31 2022", + "state": "clean", + "active_devices": "2", + "working_devices": "2", + "failed_devices": "0", + "spare_devices": "0", + "consistency_policy": "bitmap", + "name": "virttest:0", + "uuid": "85c5b164:d58a5ada:14f5fe07:d642e843", + "events": "2193679", + "device_table": [ + { + "number": "3", + "major": "8", + "minor": "17", + "state": "active sync", + "device": "/dev/sdb1", + "raid_device": "0" + }, + { + "number": "2", + "major": "8", + "minor": "33", + "state": "active sync", + "device": "/dev/sdc1", + "raid_device": "1" + } + ] + } @@ -56,7 +224,7 @@ Parameters: Returns: - List of Dictionaries. Raw or processed structured data. + Dictionary. Raw or processed structured data. ### Parser Information Compatibility: linux diff --git a/jc/parsers/mdadm.py b/jc/parsers/mdadm.py index b26df2857..52bb556bb 100644 --- a/jc/parsers/mdadm.py +++ b/jc/parsers/mdadm.py @@ -1,14 +1,22 @@ """jc - JSON Convert `mdadm` command output parser -<> +Supports the `--query` and `--examine` options in `mdadm`. Usage (cli): - $ mdadm | jc --mdadm + $ mdadm --query --detail /dev/md0 | jc --mdadm - or +or - $ jc mdadm + $ mdadm --examine -E /dev/sdb1 | jc --mdadm + +or + + $ jc mdadm --query --detail /dev/md0 + +or + + $ jc mdadm --examine -E /dev/sdb1 Usage (module): @@ -17,23 +25,183 @@ Schema: - [ - { - "mdadm": string, - "bar": boolean, - "baz": integer - } - ] + { + "device": string, + "magic": string, + "version": string, + "feature_map": string, + "array_uuid": string, + "name": string, + "name_val": string, + "uuid": string, + "uuid_val": string, + "creation_time": string, + "creation_time_epoch": integer, # naive timestamp + "raid_level": string, + "array_size": string, + "array_size_num": integer, + "used_dev_size": string, + "used_dev_size_num": integer, + "raid_devices": integer, + "avail_dev_size": string, + "avail_dev_size_num": integer, + "data_offset": integer, + "super_offset": integer, + "unused_space": string, + "unused_space_before": integer, + "unused_space_after": integer, + "state": string, + "state_list": [ + string + ], + "device_uuid": string, + "flags": string, + "flag_list": [ + string + ], + "update_time": string, + "update_time_epoch": integer, # naive timestamp + "bad_block_log": string, + "checksum": string, + "checksum_val": string, + "checksum_state": string, + "events": integer, + "chunk_size": integer, + "device_role": string, + "array_state": string, + "array_state_list": [ + string + ], + "consistency_policy": string, + "rebuild_status": string, + "rebuild_status_percent": integer, + "resync_status": string, + "resync_status_percent": integer, + "check_status": string, + "check_status_percent": integer, + "total_devices": integer, + "preferred_minor": integer, + "persistence": string, + "active_devices": integer, + "working_devices": integer, + "failed_devices": integer, + "spare_devices": integer, + "device_table": [ + { + "number": integer/null, + "major": integer/null, + "minor": integer/null, + "state": [ + string + ], + "device": string, + "raid_device": integer/null + } + ] + } Examples: - $ mdadm | jc --mdadm -p - [] + $ mdadm --query --detail /dev/md0 | jc --mdadm -p + { + "device": "/dev/md0", + "version": "1.1", + "creation_time": "Tue Apr 13 23:22:16 2010", + "raid_level": "raid1", + "array_size": "5860520828 (5.46 TiB 6.00 TB)", + "used_dev_size": "5860520828 (5.46 TiB 6.00 TB)", + "raid_devices": 2, + "total_devices": 2, + "persistence": "Superblock is persistent", + "intent_bitmap": "Internal", + "update_time": "Tue Jul 26 20:16:31 2022", + "state": "clean", + "active_devices": 2, + "working_devices": 2, + "failed_devices": 0, + "spare_devices": 0, + "consistency_policy": "bitmap", + "name": "virttest:0", + "uuid": "85c5b164:d58a5ada:14f5fe07:d642e843", + "events": 2193679, + "device_table": [ + { + "number": 3, + "major": 8, + "minor": 17, + "state": [ + "active", + "sync" + ], + "device": "/dev/sdb1", + "raid_device": 0 + }, + { + "number": 2, + "major": 8, + "minor": 33, + "state": [ + "active", + "sync" + ], + "device": "/dev/sdc1", + "raid_device": 1 + } + ], + "array_size_num": 5860520828, + "used_dev_size_num": 5860520828, + "name_val": "virttest:0", + "uuid_val": "85c5b164:d58a5ada:14f5fe07:d642e843", + "state_list": [ + "clean" + ], + "creation_time_epoch": 1271226136, + "update_time_epoch": 1658891791 + } - $ mdadm | jc --mdadm -p -r - [] + $ mdadm --query --detail /dev/md0 | jc --mdadm -p -r + { + "device": "/dev/md0", + "version": "1.1", + "creation_time": "Tue Apr 13 23:22:16 2010", + "raid_level": "raid1", + "array_size": "5860520828 (5.46 TiB 6.00 TB)", + "used_dev_size": "5860520828 (5.46 TiB 6.00 TB)", + "raid_devices": "2", + "total_devices": "2", + "persistence": "Superblock is persistent", + "intent_bitmap": "Internal", + "update_time": "Tue Jul 26 20:16:31 2022", + "state": "clean", + "active_devices": "2", + "working_devices": "2", + "failed_devices": "0", + "spare_devices": "0", + "consistency_policy": "bitmap", + "name": "virttest:0", + "uuid": "85c5b164:d58a5ada:14f5fe07:d642e843", + "events": "2193679", + "device_table": [ + { + "number": "3", + "major": "8", + "minor": "17", + "state": "active sync", + "device": "/dev/sdb1", + "raid_device": "0" + }, + { + "number": "2", + "major": "8", + "minor": "33", + "state": "active sync", + "device": "/dev/sdc1", + "raid_device": "1" + } + ] + } """ -from typing import List, Dict +from typing import Dict import jc.utils from jc.parsers.universal import sparse_table_parse @@ -57,11 +225,11 @@ def _process(proc_data: Dict) -> Dict: Parameters: - proc_data: (List of Dictionaries) raw structured data to process + proc_data: (Dictionary) raw structured data to process Returns: - List of Dictionaries. Structured to conform to the schema. + Dictionary. Structured to conform to the schema. """ int_list = {'array_size_num', 'used_dev_size_num', 'raid_devices', 'total_devices', 'active_devices', 'working_devices', 'failed_devices', 'spare_devices', @@ -180,7 +348,7 @@ def parse( Returns: - List of Dictionaries. Raw or processed structured data. + Dictionary. Raw or processed structured data. """ jc.utils.compatibility(__name__, info.compatible, quiet) jc.utils.input_type_check(data) diff --git a/man/jc.1 b/man/jc.1 index 7029fe7fe..c12fbc50d 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -1,4 +1,4 @@ -.TH jc 1 2022-08-14 1.21.0 "JSON Convert" +.TH jc 1 2022-08-15 1.21.0 "JSON Convert" .SH NAME \fBjc\fP \- JSON Convert JSONifies the output of many CLI tools and file-types .SH SYNOPSIS From 02db3baf52c9efd44f11141d1d933b534de4ffce Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Mon, 15 Aug 2022 11:14:26 -0700 Subject: [PATCH 068/116] move test fixtures --- .../examine-raid0-offline => mdadm-examine-raid0-offline.out} | 0 .../{mdadm/examine-raid0-ok => mdadm-examine-raid0-ok.out} | 0 .../examine-raid1-0.90-ok => mdadm-examine-raid1-0-90-ok.out} | 0 .../examine-raid1-checking => mdadm-examine-raid1-checking.out} | 0 .../examine-raid1-failfast => mdadm-examine-raid1-failfast.out} | 0 .../examine-raid1-faulty1 => mdadm-examine-raid1-faulty1.out} | 0 .../examine-raid1-faulty2 => mdadm-examine-raid1-faulty2.out} | 0 .../examine-raid1-moreflags => mdadm-examine-raid1-moreflags.out} | 0 .../{mdadm/examine-raid1-ok => mdadm-examine-raid1-ok.out} | 0 .../examine-raid1-replacing => mdadm-examine-raid1-replacing.out} | 0 .../examine-raid1-resync => mdadm-examine-raid1-resync.out} | 0 .../{mdadm/examine-raid1-spare => mdadm-examine-raid1-spare.out} | 0 .../examine-raid1-syncing => mdadm-examine-raid1-syncing.out} | 0 tests/fixtures/generic/{mdadm => }/mdadm-examine.out | 0 tests/fixtures/generic/{mdadm => }/mdadm-query-detail.out | 0 .../generic/{mdadm/query-raid0-ok => mdadm-query-raid0-ok.out} | 0 ...d1-failed-and-flags => mdadm-query-raid1-failed-and-flags.out} | 0 ...aulty_and_removed => mdadm-query-raid1-faulty-and-removed.out} | 0 ...-faulty-with-spare => mdadm-query-raid1-faulty-with-spare.out} | 0 .../{mdadm/query-raid1-faulty => mdadm-query-raid1-faulty.out} | 0 .../{mdadm/query-raid1-ok-0.9 => mdadm-query-raid1-ok-0-9.out} | 0 .../query-raid1-ok-failfast => mdadm-query-raid1-ok-failfast.out} | 0 .../query-raid1-ok-spare => mdadm-query-raid1-ok-spare.out} | 0 .../generic/{mdadm/query-raid1-ok => mdadm-query-raid1-ok.out} | 0 ...d1-rebuild-failfast => mdadm-query-raid1-rebuild-failfast.out} | 0 ...-writem-rebuild => mdadm-query-raid1-spare-writem-rebuild.out} | 0 .../{mdadm/query-raid1-syncing => mdadm-query-raid1-syncing.out} | 0 27 files changed, 0 insertions(+), 0 deletions(-) rename tests/fixtures/generic/{mdadm/examine-raid0-offline => mdadm-examine-raid0-offline.out} (100%) rename tests/fixtures/generic/{mdadm/examine-raid0-ok => mdadm-examine-raid0-ok.out} (100%) rename tests/fixtures/generic/{mdadm/examine-raid1-0.90-ok => mdadm-examine-raid1-0-90-ok.out} (100%) rename tests/fixtures/generic/{mdadm/examine-raid1-checking => mdadm-examine-raid1-checking.out} (100%) rename tests/fixtures/generic/{mdadm/examine-raid1-failfast => mdadm-examine-raid1-failfast.out} (100%) rename tests/fixtures/generic/{mdadm/examine-raid1-faulty1 => mdadm-examine-raid1-faulty1.out} (100%) rename tests/fixtures/generic/{mdadm/examine-raid1-faulty2 => mdadm-examine-raid1-faulty2.out} (100%) rename tests/fixtures/generic/{mdadm/examine-raid1-moreflags => mdadm-examine-raid1-moreflags.out} (100%) rename tests/fixtures/generic/{mdadm/examine-raid1-ok => mdadm-examine-raid1-ok.out} (100%) rename tests/fixtures/generic/{mdadm/examine-raid1-replacing => mdadm-examine-raid1-replacing.out} (100%) rename tests/fixtures/generic/{mdadm/examine-raid1-resync => mdadm-examine-raid1-resync.out} (100%) rename tests/fixtures/generic/{mdadm/examine-raid1-spare => mdadm-examine-raid1-spare.out} (100%) rename tests/fixtures/generic/{mdadm/examine-raid1-syncing => mdadm-examine-raid1-syncing.out} (100%) rename tests/fixtures/generic/{mdadm => }/mdadm-examine.out (100%) rename tests/fixtures/generic/{mdadm => }/mdadm-query-detail.out (100%) rename tests/fixtures/generic/{mdadm/query-raid0-ok => mdadm-query-raid0-ok.out} (100%) rename tests/fixtures/generic/{mdadm/query-raid1-failed-and-flags => mdadm-query-raid1-failed-and-flags.out} (100%) rename tests/fixtures/generic/{mdadm/query-raid1-faulty_and_removed => mdadm-query-raid1-faulty-and-removed.out} (100%) rename tests/fixtures/generic/{mdadm/query-raid1-faulty-with-spare => mdadm-query-raid1-faulty-with-spare.out} (100%) rename tests/fixtures/generic/{mdadm/query-raid1-faulty => mdadm-query-raid1-faulty.out} (100%) rename tests/fixtures/generic/{mdadm/query-raid1-ok-0.9 => mdadm-query-raid1-ok-0-9.out} (100%) rename tests/fixtures/generic/{mdadm/query-raid1-ok-failfast => mdadm-query-raid1-ok-failfast.out} (100%) rename tests/fixtures/generic/{mdadm/query-raid1-ok-spare => mdadm-query-raid1-ok-spare.out} (100%) rename tests/fixtures/generic/{mdadm/query-raid1-ok => mdadm-query-raid1-ok.out} (100%) rename tests/fixtures/generic/{mdadm/query-raid1-rebuild-failfast => mdadm-query-raid1-rebuild-failfast.out} (100%) rename tests/fixtures/generic/{mdadm/query-raid1-spare-writem-rebuild => mdadm-query-raid1-spare-writem-rebuild.out} (100%) rename tests/fixtures/generic/{mdadm/query-raid1-syncing => mdadm-query-raid1-syncing.out} (100%) diff --git a/tests/fixtures/generic/mdadm/examine-raid0-offline b/tests/fixtures/generic/mdadm-examine-raid0-offline.out similarity index 100% rename from tests/fixtures/generic/mdadm/examine-raid0-offline rename to tests/fixtures/generic/mdadm-examine-raid0-offline.out diff --git a/tests/fixtures/generic/mdadm/examine-raid0-ok b/tests/fixtures/generic/mdadm-examine-raid0-ok.out similarity index 100% rename from tests/fixtures/generic/mdadm/examine-raid0-ok rename to tests/fixtures/generic/mdadm-examine-raid0-ok.out diff --git a/tests/fixtures/generic/mdadm/examine-raid1-0.90-ok b/tests/fixtures/generic/mdadm-examine-raid1-0-90-ok.out similarity index 100% rename from tests/fixtures/generic/mdadm/examine-raid1-0.90-ok rename to tests/fixtures/generic/mdadm-examine-raid1-0-90-ok.out diff --git a/tests/fixtures/generic/mdadm/examine-raid1-checking b/tests/fixtures/generic/mdadm-examine-raid1-checking.out similarity index 100% rename from tests/fixtures/generic/mdadm/examine-raid1-checking rename to tests/fixtures/generic/mdadm-examine-raid1-checking.out diff --git a/tests/fixtures/generic/mdadm/examine-raid1-failfast b/tests/fixtures/generic/mdadm-examine-raid1-failfast.out similarity index 100% rename from tests/fixtures/generic/mdadm/examine-raid1-failfast rename to tests/fixtures/generic/mdadm-examine-raid1-failfast.out diff --git a/tests/fixtures/generic/mdadm/examine-raid1-faulty1 b/tests/fixtures/generic/mdadm-examine-raid1-faulty1.out similarity index 100% rename from tests/fixtures/generic/mdadm/examine-raid1-faulty1 rename to tests/fixtures/generic/mdadm-examine-raid1-faulty1.out diff --git a/tests/fixtures/generic/mdadm/examine-raid1-faulty2 b/tests/fixtures/generic/mdadm-examine-raid1-faulty2.out similarity index 100% rename from tests/fixtures/generic/mdadm/examine-raid1-faulty2 rename to tests/fixtures/generic/mdadm-examine-raid1-faulty2.out diff --git a/tests/fixtures/generic/mdadm/examine-raid1-moreflags b/tests/fixtures/generic/mdadm-examine-raid1-moreflags.out similarity index 100% rename from tests/fixtures/generic/mdadm/examine-raid1-moreflags rename to tests/fixtures/generic/mdadm-examine-raid1-moreflags.out diff --git a/tests/fixtures/generic/mdadm/examine-raid1-ok b/tests/fixtures/generic/mdadm-examine-raid1-ok.out similarity index 100% rename from tests/fixtures/generic/mdadm/examine-raid1-ok rename to tests/fixtures/generic/mdadm-examine-raid1-ok.out diff --git a/tests/fixtures/generic/mdadm/examine-raid1-replacing b/tests/fixtures/generic/mdadm-examine-raid1-replacing.out similarity index 100% rename from tests/fixtures/generic/mdadm/examine-raid1-replacing rename to tests/fixtures/generic/mdadm-examine-raid1-replacing.out diff --git a/tests/fixtures/generic/mdadm/examine-raid1-resync b/tests/fixtures/generic/mdadm-examine-raid1-resync.out similarity index 100% rename from tests/fixtures/generic/mdadm/examine-raid1-resync rename to tests/fixtures/generic/mdadm-examine-raid1-resync.out diff --git a/tests/fixtures/generic/mdadm/examine-raid1-spare b/tests/fixtures/generic/mdadm-examine-raid1-spare.out similarity index 100% rename from tests/fixtures/generic/mdadm/examine-raid1-spare rename to tests/fixtures/generic/mdadm-examine-raid1-spare.out diff --git a/tests/fixtures/generic/mdadm/examine-raid1-syncing b/tests/fixtures/generic/mdadm-examine-raid1-syncing.out similarity index 100% rename from tests/fixtures/generic/mdadm/examine-raid1-syncing rename to tests/fixtures/generic/mdadm-examine-raid1-syncing.out diff --git a/tests/fixtures/generic/mdadm/mdadm-examine.out b/tests/fixtures/generic/mdadm-examine.out similarity index 100% rename from tests/fixtures/generic/mdadm/mdadm-examine.out rename to tests/fixtures/generic/mdadm-examine.out diff --git a/tests/fixtures/generic/mdadm/mdadm-query-detail.out b/tests/fixtures/generic/mdadm-query-detail.out similarity index 100% rename from tests/fixtures/generic/mdadm/mdadm-query-detail.out rename to tests/fixtures/generic/mdadm-query-detail.out diff --git a/tests/fixtures/generic/mdadm/query-raid0-ok b/tests/fixtures/generic/mdadm-query-raid0-ok.out similarity index 100% rename from tests/fixtures/generic/mdadm/query-raid0-ok rename to tests/fixtures/generic/mdadm-query-raid0-ok.out diff --git a/tests/fixtures/generic/mdadm/query-raid1-failed-and-flags b/tests/fixtures/generic/mdadm-query-raid1-failed-and-flags.out similarity index 100% rename from tests/fixtures/generic/mdadm/query-raid1-failed-and-flags rename to tests/fixtures/generic/mdadm-query-raid1-failed-and-flags.out diff --git a/tests/fixtures/generic/mdadm/query-raid1-faulty_and_removed b/tests/fixtures/generic/mdadm-query-raid1-faulty-and-removed.out similarity index 100% rename from tests/fixtures/generic/mdadm/query-raid1-faulty_and_removed rename to tests/fixtures/generic/mdadm-query-raid1-faulty-and-removed.out diff --git a/tests/fixtures/generic/mdadm/query-raid1-faulty-with-spare b/tests/fixtures/generic/mdadm-query-raid1-faulty-with-spare.out similarity index 100% rename from tests/fixtures/generic/mdadm/query-raid1-faulty-with-spare rename to tests/fixtures/generic/mdadm-query-raid1-faulty-with-spare.out diff --git a/tests/fixtures/generic/mdadm/query-raid1-faulty b/tests/fixtures/generic/mdadm-query-raid1-faulty.out similarity index 100% rename from tests/fixtures/generic/mdadm/query-raid1-faulty rename to tests/fixtures/generic/mdadm-query-raid1-faulty.out diff --git a/tests/fixtures/generic/mdadm/query-raid1-ok-0.9 b/tests/fixtures/generic/mdadm-query-raid1-ok-0-9.out similarity index 100% rename from tests/fixtures/generic/mdadm/query-raid1-ok-0.9 rename to tests/fixtures/generic/mdadm-query-raid1-ok-0-9.out diff --git a/tests/fixtures/generic/mdadm/query-raid1-ok-failfast b/tests/fixtures/generic/mdadm-query-raid1-ok-failfast.out similarity index 100% rename from tests/fixtures/generic/mdadm/query-raid1-ok-failfast rename to tests/fixtures/generic/mdadm-query-raid1-ok-failfast.out diff --git a/tests/fixtures/generic/mdadm/query-raid1-ok-spare b/tests/fixtures/generic/mdadm-query-raid1-ok-spare.out similarity index 100% rename from tests/fixtures/generic/mdadm/query-raid1-ok-spare rename to tests/fixtures/generic/mdadm-query-raid1-ok-spare.out diff --git a/tests/fixtures/generic/mdadm/query-raid1-ok b/tests/fixtures/generic/mdadm-query-raid1-ok.out similarity index 100% rename from tests/fixtures/generic/mdadm/query-raid1-ok rename to tests/fixtures/generic/mdadm-query-raid1-ok.out diff --git a/tests/fixtures/generic/mdadm/query-raid1-rebuild-failfast b/tests/fixtures/generic/mdadm-query-raid1-rebuild-failfast.out similarity index 100% rename from tests/fixtures/generic/mdadm/query-raid1-rebuild-failfast rename to tests/fixtures/generic/mdadm-query-raid1-rebuild-failfast.out diff --git a/tests/fixtures/generic/mdadm/query-raid1-spare-writem-rebuild b/tests/fixtures/generic/mdadm-query-raid1-spare-writem-rebuild.out similarity index 100% rename from tests/fixtures/generic/mdadm/query-raid1-spare-writem-rebuild rename to tests/fixtures/generic/mdadm-query-raid1-spare-writem-rebuild.out diff --git a/tests/fixtures/generic/mdadm/query-raid1-syncing b/tests/fixtures/generic/mdadm-query-raid1-syncing.out similarity index 100% rename from tests/fixtures/generic/mdadm/query-raid1-syncing rename to tests/fixtures/generic/mdadm-query-raid1-syncing.out From 3daf90c5fe95611521305344aa2ac1f6a0018fd9 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Mon, 15 Aug 2022 13:51:48 -0700 Subject: [PATCH 069/116] doc formatting --- README.md | 2 +- completions/jc_zsh_completion.sh | 2 +- docs/parsers/acpi.md | 2 +- docs/parsers/airport.md | 2 +- docs/parsers/airport_s.md | 2 +- docs/parsers/arp.md | 2 +- docs/parsers/blkid.md | 2 +- docs/parsers/chage.md | 2 +- docs/parsers/cksum.md | 2 +- docs/parsers/crontab.md | 2 +- docs/parsers/date.md | 2 +- docs/parsers/df.md | 2 +- docs/parsers/dig.md | 2 +- docs/parsers/dmidecode.md | 2 +- docs/parsers/dpkg_l.md | 2 +- docs/parsers/du.md | 2 +- docs/parsers/env.md | 2 +- docs/parsers/file.md | 2 +- docs/parsers/finger.md | 2 +- docs/parsers/free.md | 2 +- docs/parsers/git_log.md | 2 +- docs/parsers/gpg.md | 2 +- docs/parsers/hashsum.md | 2 +- docs/parsers/hciconfig.md | 2 +- docs/parsers/id.md | 2 +- docs/parsers/ifconfig.md | 2 +- docs/parsers/iostat.md | 2 +- docs/parsers/iptables.md | 2 +- docs/parsers/iw_scan.md | 2 +- docs/parsers/jar_manifest.md | 2 +- docs/parsers/last.md | 2 +- docs/parsers/ls.md | 2 +- docs/parsers/lsblk.md | 2 +- docs/parsers/lsmod.md | 2 +- docs/parsers/lsof.md | 2 +- docs/parsers/lsusb.md | 2 +- docs/parsers/mount.md | 2 +- docs/parsers/mpstat.md | 2 +- docs/parsers/netstat.md | 2 +- docs/parsers/nmcli.md | 2 +- docs/parsers/ntpq.md | 2 +- docs/parsers/pidstat.md | 2 +- docs/parsers/ping.md | 2 +- docs/parsers/pip_list.md | 2 +- docs/parsers/pip_show.md | 2 +- docs/parsers/postconf.md | 2 +- docs/parsers/ps.md | 2 +- docs/parsers/route.md | 2 +- docs/parsers/rpm_qi.md | 2 +- docs/parsers/rsync.md | 4 ++-- docs/parsers/rsync_s.md | 2 +- docs/parsers/sfdisk.md | 2 +- docs/parsers/ss.md | 2 +- docs/parsers/stat.md | 2 +- docs/parsers/sysctl.md | 2 +- docs/parsers/syslog_bsd.md | 2 +- docs/parsers/systemctl.md | 2 +- docs/parsers/systemctl_lj.md | 2 +- docs/parsers/systemctl_ls.md | 2 +- docs/parsers/systemctl_luf.md | 2 +- docs/parsers/timedatectl.md | 2 +- docs/parsers/top.md | 2 +- docs/parsers/tracepath.md | 2 +- docs/parsers/traceroute.md | 2 +- docs/parsers/ufw.md | 2 +- docs/parsers/ufw_appinfo.md | 2 +- docs/parsers/uname.md | 2 +- docs/parsers/update_alt_gs.md | 2 +- docs/parsers/update_alt_q.md | 2 +- docs/parsers/upower.md | 2 +- docs/parsers/uptime.md | 2 +- docs/parsers/vmstat.md | 2 +- docs/parsers/w.md | 2 +- docs/parsers/wc.md | 2 +- docs/parsers/who.md | 2 +- docs/parsers/xrandr.md | 2 +- docs/parsers/zipinfo.md | 2 +- jc/parsers/acpi.py | 2 +- jc/parsers/airport.py | 2 +- jc/parsers/airport_s.py | 2 +- jc/parsers/arp.py | 2 +- jc/parsers/blkid.py | 2 +- jc/parsers/cef.py | 11 +++++++---- jc/parsers/chage.py | 2 +- jc/parsers/cksum.py | 2 +- jc/parsers/crontab.py | 2 +- jc/parsers/date.py | 2 +- jc/parsers/df.py | 2 +- jc/parsers/dig.py | 2 +- jc/parsers/dmidecode.py | 2 +- jc/parsers/dpkg_l.py | 2 +- jc/parsers/du.py | 2 +- jc/parsers/env.py | 2 +- jc/parsers/file.py | 2 +- jc/parsers/finger.py | 2 +- jc/parsers/foo.py | 2 +- jc/parsers/free.py | 2 +- jc/parsers/git_log.py | 2 +- jc/parsers/gpg.py | 2 +- jc/parsers/hashsum.py | 2 +- jc/parsers/hciconfig.py | 2 +- jc/parsers/id.py | 2 +- jc/parsers/ifconfig.py | 2 +- jc/parsers/iostat.py | 2 +- jc/parsers/iptables.py | 2 +- jc/parsers/iw_scan.py | 2 +- jc/parsers/jar_manifest.py | 4 ++-- jc/parsers/last.py | 2 +- jc/parsers/ls.py | 2 +- jc/parsers/lsblk.py | 2 +- jc/parsers/lsmod.py | 2 +- jc/parsers/lsof.py | 2 +- jc/parsers/lsusb.py | 2 +- jc/parsers/mount.py | 2 +- jc/parsers/mpstat.py | 2 +- jc/parsers/netstat.py | 2 +- jc/parsers/nmcli.py | 2 +- jc/parsers/ntpq.py | 2 +- jc/parsers/pidstat.py | 2 +- jc/parsers/ping.py | 2 +- jc/parsers/pip_list.py | 2 +- jc/parsers/pip_show.py | 2 +- jc/parsers/postconf.py | 2 +- jc/parsers/ps.py | 2 +- jc/parsers/route.py | 2 +- jc/parsers/rpm_qi.py | 2 +- jc/parsers/rsync.py | 4 ++-- jc/parsers/rsync_s.py | 2 +- jc/parsers/sfdisk.py | 2 +- jc/parsers/ss.py | 2 +- jc/parsers/stat.py | 2 +- jc/parsers/sysctl.py | 2 +- jc/parsers/syslog_bsd.py | 2 +- jc/parsers/systemctl.py | 2 +- jc/parsers/systemctl_lj.py | 2 +- jc/parsers/systemctl_ls.py | 2 +- jc/parsers/systemctl_luf.py | 2 +- jc/parsers/timedatectl.py | 2 +- jc/parsers/top.py | 2 +- jc/parsers/tracepath.py | 2 +- jc/parsers/traceroute.py | 2 +- jc/parsers/ufw.py | 2 +- jc/parsers/ufw_appinfo.py | 2 +- jc/parsers/uname.py | 2 +- jc/parsers/update_alt_gs.py | 2 +- jc/parsers/update_alt_q.py | 2 +- jc/parsers/upower.py | 2 +- jc/parsers/uptime.py | 2 +- jc/parsers/vmstat.py | 2 +- jc/parsers/w.py | 2 +- jc/parsers/wc.py | 2 +- jc/parsers/who.py | 2 +- jc/parsers/xrandr.py | 2 +- jc/parsers/zipinfo.py | 2 +- man/jc.1 | 2 +- 155 files changed, 164 insertions(+), 161 deletions(-) diff --git a/README.md b/README.md index 98ab3a51e..d3bd1f6da 100644 --- a/README.md +++ b/README.md @@ -196,7 +196,7 @@ option. | ` --iptables` | `iptables` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/iptables) | | ` --iso-datetime` | ISO 8601 Datetime string parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/iso_datetime) | | ` --iw-scan` | `iw dev [device] scan` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/iw_scan) | -| ` --jar-manifest` | MANIFEST.MF file parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/jar_manifest) | +| ` --jar-manifest` | Java MANIFEST.MF file parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/jar_manifest) | | ` --jobs` | `jobs` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/jobs) | | ` --jwt` | JWT string parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/jwt) | | ` --kv` | Key/Value file parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/kv) | diff --git a/completions/jc_zsh_completion.sh b/completions/jc_zsh_completion.sh index 60b2ee09e..c527cd31f 100644 --- a/completions/jc_zsh_completion.sh +++ b/completions/jc_zsh_completion.sh @@ -143,7 +143,7 @@ _jc() { '--iptables:`iptables` command parser' '--iso-datetime:ISO 8601 Datetime string parser' '--iw-scan:`iw dev [device] scan` command parser' - '--jar-manifest:MANIFEST.MF file parser' + '--jar-manifest:Java MANIFEST.MF file parser' '--jobs:`jobs` command parser' '--jwt:JWT string parser' '--kv:Key/Value file parser' diff --git a/docs/parsers/acpi.md b/docs/parsers/acpi.md index 56da4d314..096db5812 100644 --- a/docs/parsers/acpi.md +++ b/docs/parsers/acpi.md @@ -9,7 +9,7 @@ Usage (cli): $ acpi -V | jc --acpi - or +or $ jc acpi -V diff --git a/docs/parsers/airport.md b/docs/parsers/airport.md index bf934cfab..bd29299e8 100644 --- a/docs/parsers/airport.md +++ b/docs/parsers/airport.md @@ -11,7 +11,7 @@ Usage (cli): $ airport -I | jc --airport - or +or $ jc airport -I diff --git a/docs/parsers/airport_s.md b/docs/parsers/airport_s.md index f9eac73c6..931f808d9 100644 --- a/docs/parsers/airport_s.md +++ b/docs/parsers/airport_s.md @@ -11,7 +11,7 @@ Usage (cli): $ airport -s | jc --airport-s - or +or $ jc airport -s diff --git a/docs/parsers/arp.md b/docs/parsers/arp.md index 18c31624f..59750bf38 100644 --- a/docs/parsers/arp.md +++ b/docs/parsers/arp.md @@ -11,7 +11,7 @@ Usage (cli): $ arp | jc --arp - or +or $ jc arp diff --git a/docs/parsers/blkid.md b/docs/parsers/blkid.md index e1d138004..d3580996f 100644 --- a/docs/parsers/blkid.md +++ b/docs/parsers/blkid.md @@ -9,7 +9,7 @@ Usage (cli): $ blkid | jc --blkid - or +or $ jc blkid diff --git a/docs/parsers/chage.md b/docs/parsers/chage.md index ef895f56f..5b13f29a0 100644 --- a/docs/parsers/chage.md +++ b/docs/parsers/chage.md @@ -11,7 +11,7 @@ Usage (cli): $ chage -l johndoe | jc --chage - or +or $ jc chage -l johndoe diff --git a/docs/parsers/cksum.md b/docs/parsers/cksum.md index bebd8b56b..c09a06eee 100644 --- a/docs/parsers/cksum.md +++ b/docs/parsers/cksum.md @@ -13,7 +13,7 @@ Usage (cli): $ cksum file.txt | jc --cksum - or +or $ jc cksum file.txt diff --git a/docs/parsers/crontab.md b/docs/parsers/crontab.md index 74b91cdf5..35e40d801 100644 --- a/docs/parsers/crontab.md +++ b/docs/parsers/crontab.md @@ -12,7 +12,7 @@ Usage (cli): $ crontab -l | jc --crontab - or +or $ jc crontab -l diff --git a/docs/parsers/date.md b/docs/parsers/date.md index 3ab72120a..374cb9c77 100644 --- a/docs/parsers/date.md +++ b/docs/parsers/date.md @@ -15,7 +15,7 @@ Usage (cli): $ date | jc --date - or +or $ jc date diff --git a/docs/parsers/df.md b/docs/parsers/df.md index 8abe56a7d..cebd20ba3 100644 --- a/docs/parsers/df.md +++ b/docs/parsers/df.md @@ -9,7 +9,7 @@ Usage (cli): $ df | jc --df - or +or $ jc df diff --git a/docs/parsers/dig.md b/docs/parsers/dig.md index 7e2042568..27bcdd6f4 100644 --- a/docs/parsers/dig.md +++ b/docs/parsers/dig.md @@ -20,7 +20,7 @@ Usage (cli): $ dig example.com | jc --dig - or +or $ jc dig example.com diff --git a/docs/parsers/dmidecode.md b/docs/parsers/dmidecode.md index 33f10ac19..797f965d6 100644 --- a/docs/parsers/dmidecode.md +++ b/docs/parsers/dmidecode.md @@ -9,7 +9,7 @@ Usage (cli): $ dmidecode | jc --dmidecode - or +or $ jc dmidecode diff --git a/docs/parsers/dpkg_l.md b/docs/parsers/dpkg_l.md index 51e64f46f..e5ce50710 100644 --- a/docs/parsers/dpkg_l.md +++ b/docs/parsers/dpkg_l.md @@ -14,7 +14,7 @@ Usage (cli): $ dpkg -l | jc --dpkg-l - or +or $ jc dpkg -l diff --git a/docs/parsers/du.md b/docs/parsers/du.md index e5bc86f59..d9bce12db 100644 --- a/docs/parsers/du.md +++ b/docs/parsers/du.md @@ -9,7 +9,7 @@ Usage (cli): $ du | jc --du - or +or $ jc du diff --git a/docs/parsers/env.md b/docs/parsers/env.md index 89d10adac..02f406b68 100644 --- a/docs/parsers/env.md +++ b/docs/parsers/env.md @@ -14,7 +14,7 @@ Usage (cli): $ env | jc --env - or +or $ jc env diff --git a/docs/parsers/file.md b/docs/parsers/file.md index 71ab2b8e8..d1498c26f 100644 --- a/docs/parsers/file.md +++ b/docs/parsers/file.md @@ -9,7 +9,7 @@ Usage (cli): $ file * | jc --file - or +or $ jc file * diff --git a/docs/parsers/finger.md b/docs/parsers/finger.md index 438bba05e..f889bcadd 100644 --- a/docs/parsers/finger.md +++ b/docs/parsers/finger.md @@ -11,7 +11,7 @@ Usage (cli): $ finger | jc --finger - or +or $ jc finger diff --git a/docs/parsers/free.md b/docs/parsers/free.md index a2d7dc1b0..1e9fb92f0 100644 --- a/docs/parsers/free.md +++ b/docs/parsers/free.md @@ -9,7 +9,7 @@ Usage (cli): $ free | jc --free - or +or $ jc free diff --git a/docs/parsers/git_log.md b/docs/parsers/git_log.md index 0064e5fcd..44ae89aff 100644 --- a/docs/parsers/git_log.md +++ b/docs/parsers/git_log.md @@ -26,7 +26,7 @@ Usage (cli): $ git log | jc --git-log - or +or $ jc git log diff --git a/docs/parsers/gpg.md b/docs/parsers/gpg.md index c3fba80be..6fd7bf2a7 100644 --- a/docs/parsers/gpg.md +++ b/docs/parsers/gpg.md @@ -9,7 +9,7 @@ Usage (cli): $ gpg --with-colons --show-keys file.gpg | jc --gpg - or +or $ jc gpg --with-colons --show-keys file.gpg diff --git a/docs/parsers/hashsum.md b/docs/parsers/hashsum.md index e139e98a8..8cbbc6ff5 100644 --- a/docs/parsers/hashsum.md +++ b/docs/parsers/hashsum.md @@ -19,7 +19,7 @@ Usage (cli): $ md5sum file.txt | jc --hashsum - or +or $ jc md5sum file.txt diff --git a/docs/parsers/hciconfig.md b/docs/parsers/hciconfig.md index 26a32f9d9..342c7b998 100644 --- a/docs/parsers/hciconfig.md +++ b/docs/parsers/hciconfig.md @@ -9,7 +9,7 @@ Usage (cli): $ hciconfig | jc --hciconfig - or +or $ jc hciconfig diff --git a/docs/parsers/id.md b/docs/parsers/id.md index 976e0cf3f..eaf325549 100644 --- a/docs/parsers/id.md +++ b/docs/parsers/id.md @@ -9,7 +9,7 @@ Usage (cli): $ id | jc --id - or +or $ jc id diff --git a/docs/parsers/ifconfig.md b/docs/parsers/ifconfig.md index 9026da4c5..ac2d6adb2 100644 --- a/docs/parsers/ifconfig.md +++ b/docs/parsers/ifconfig.md @@ -11,7 +11,7 @@ Usage (cli): $ ifconfig | jc --ifconfig - or +or $ jc ifconfig diff --git a/docs/parsers/iostat.md b/docs/parsers/iostat.md index ecab5b9b1..796069196 100644 --- a/docs/parsers/iostat.md +++ b/docs/parsers/iostat.md @@ -11,7 +11,7 @@ Usage (cli): $ iostat | jc --iostat - or +or $ jc iostat diff --git a/docs/parsers/iptables.md b/docs/parsers/iptables.md index bc646347e..7c9f7d8f6 100644 --- a/docs/parsers/iptables.md +++ b/docs/parsers/iptables.md @@ -11,7 +11,7 @@ Usage (cli): $ sudo iptables -L -t nat | jc --iptables - or +or $ jc iptables -L -t nat diff --git a/docs/parsers/iw_scan.md b/docs/parsers/iw_scan.md index 48d566afb..a38b0aa2d 100644 --- a/docs/parsers/iw_scan.md +++ b/docs/parsers/iw_scan.md @@ -12,7 +12,7 @@ Usage (cli): $ iw dev wlan0 scan | jc --iw-scan - or +or $ jc iw dev wlan0 scan diff --git a/docs/parsers/jar_manifest.md b/docs/parsers/jar_manifest.md index 45170037a..bdaf6d40c 100644 --- a/docs/parsers/jar_manifest.md +++ b/docs/parsers/jar_manifest.md @@ -3,7 +3,7 @@ # jc.parsers.jar\_manifest -jc - JSON Convert `MANIFEST.MF` file parser +jc - JSON Convert Java `MANIFEST.MF` file parser Usage (cli): diff --git a/docs/parsers/last.md b/docs/parsers/last.md index 012946d94..64927d500 100644 --- a/docs/parsers/last.md +++ b/docs/parsers/last.md @@ -15,7 +15,7 @@ Usage (cli): $ last | jc --last - or +or $ jc last diff --git a/docs/parsers/ls.md b/docs/parsers/ls.md index a720b1159..02c6edd69 100644 --- a/docs/parsers/ls.md +++ b/docs/parsers/ls.md @@ -26,7 +26,7 @@ Usage (cli): $ ls | jc --ls - or +or $ jc ls diff --git a/docs/parsers/lsblk.md b/docs/parsers/lsblk.md index 95acfdfbb..3f2494150 100644 --- a/docs/parsers/lsblk.md +++ b/docs/parsers/lsblk.md @@ -9,7 +9,7 @@ Usage (cli): $ lsblk | jc --lsblk - or +or $ jc lsblk diff --git a/docs/parsers/lsmod.md b/docs/parsers/lsmod.md index d73b720a1..ac657f8d5 100644 --- a/docs/parsers/lsmod.md +++ b/docs/parsers/lsmod.md @@ -9,7 +9,7 @@ Usage (cli): $ lsmod | jc --lsmod - or +or $ jc lsmod diff --git a/docs/parsers/lsof.md b/docs/parsers/lsof.md index aeb588003..6a0f7945f 100644 --- a/docs/parsers/lsof.md +++ b/docs/parsers/lsof.md @@ -9,7 +9,7 @@ Usage (cli): $ lsof | jc --lsof - or +or $ jc lsof diff --git a/docs/parsers/lsusb.md b/docs/parsers/lsusb.md index c6d067a61..2102e872c 100644 --- a/docs/parsers/lsusb.md +++ b/docs/parsers/lsusb.md @@ -11,7 +11,7 @@ Usage (cli): $ lsusb -v | jc --lsusb - or +or $ jc lsusb -v diff --git a/docs/parsers/mount.md b/docs/parsers/mount.md index 39dcc1e76..0aa06d271 100644 --- a/docs/parsers/mount.md +++ b/docs/parsers/mount.md @@ -9,7 +9,7 @@ Usage (cli): $ mount | jc --mount - or +or $ jc mount diff --git a/docs/parsers/mpstat.md b/docs/parsers/mpstat.md index f2365b71d..33e5e989c 100644 --- a/docs/parsers/mpstat.md +++ b/docs/parsers/mpstat.md @@ -11,7 +11,7 @@ Usage (cli): $ mpstat | jc --mpstat - or +or $ jc mpstat diff --git a/docs/parsers/netstat.md b/docs/parsers/netstat.md index 50b29032c..beb06e781 100644 --- a/docs/parsers/netstat.md +++ b/docs/parsers/netstat.md @@ -14,7 +14,7 @@ Usage (cli): $ netstat | jc --netstat - or +or $ jc netstat diff --git a/docs/parsers/nmcli.md b/docs/parsers/nmcli.md index bdb50fcb0..79948d094 100644 --- a/docs/parsers/nmcli.md +++ b/docs/parsers/nmcli.md @@ -18,7 +18,7 @@ Usage (cli): $ nmcli device show lo | jc --nmcli - or +or $ jc nmcli device show lo diff --git a/docs/parsers/ntpq.md b/docs/parsers/ntpq.md index e3b447667..a35d4b52a 100644 --- a/docs/parsers/ntpq.md +++ b/docs/parsers/ntpq.md @@ -9,7 +9,7 @@ Usage (cli): $ ntpq -p | jc --ntpq - or +or $ jc ntpq -p diff --git a/docs/parsers/pidstat.md b/docs/parsers/pidstat.md index 056382c12..e8d97af54 100644 --- a/docs/parsers/pidstat.md +++ b/docs/parsers/pidstat.md @@ -12,7 +12,7 @@ Usage (cli): $ pidstat -h | jc --pidstat - or +or $ jc pidstat -h diff --git a/docs/parsers/ping.md b/docs/parsers/ping.md index c0675ab45..e13e41b1d 100644 --- a/docs/parsers/ping.md +++ b/docs/parsers/ping.md @@ -14,7 +14,7 @@ Usage (cli): $ ping -c 3 1.2.3.4 | jc --ping - or +or $ jc ping -c 3 1.2.3.4 diff --git a/docs/parsers/pip_list.md b/docs/parsers/pip_list.md index eb3952405..8f026aaa8 100644 --- a/docs/parsers/pip_list.md +++ b/docs/parsers/pip_list.md @@ -9,7 +9,7 @@ Usage (cli): $ pip list | jc --pip-list - or +or $ jc pip list diff --git a/docs/parsers/pip_show.md b/docs/parsers/pip_show.md index 41c0cf4d0..6ff259733 100644 --- a/docs/parsers/pip_show.md +++ b/docs/parsers/pip_show.md @@ -9,7 +9,7 @@ Usage (cli): $ pip show | jc --pip-show - or +or $ jc pip show diff --git a/docs/parsers/postconf.md b/docs/parsers/postconf.md index 2ef586daf..aa069f0fb 100644 --- a/docs/parsers/postconf.md +++ b/docs/parsers/postconf.md @@ -9,7 +9,7 @@ Usage (cli): $ postconf -M | jc --postconf - or +or $ jc postconf -M diff --git a/docs/parsers/ps.md b/docs/parsers/ps.md index cef7237d7..749bc0064 100644 --- a/docs/parsers/ps.md +++ b/docs/parsers/ps.md @@ -13,7 +13,7 @@ Usage (cli): $ ps | jc --ps - or +or $ jc ps diff --git a/docs/parsers/route.md b/docs/parsers/route.md index 60b932e3b..3569de485 100644 --- a/docs/parsers/route.md +++ b/docs/parsers/route.md @@ -9,7 +9,7 @@ Usage (cli): $ route | jc --route - or +or $ jc route diff --git a/docs/parsers/rpm_qi.md b/docs/parsers/rpm_qi.md index 4e77dc0e2..a2ec798ac 100644 --- a/docs/parsers/rpm_qi.md +++ b/docs/parsers/rpm_qi.md @@ -17,7 +17,7 @@ Usage (cli): $ rpm -qia | jc --rpm-qi - or +or $ jc rpm -qia diff --git a/docs/parsers/rsync.md b/docs/parsers/rsync.md index 4040d6cae..9cdf7db92 100644 --- a/docs/parsers/rsync.md +++ b/docs/parsers/rsync.md @@ -13,11 +13,11 @@ Usage (cli): $ rsync -i -a source/ dest | jc --rsync - or +or $ jc rsync -i -a source/ dest - or +or $ cat rsync-backup.log | jc --rsync diff --git a/docs/parsers/rsync_s.md b/docs/parsers/rsync_s.md index 2eaa54c02..20c184207 100644 --- a/docs/parsers/rsync_s.md +++ b/docs/parsers/rsync_s.md @@ -16,7 +16,7 @@ Usage (cli): $ rsync -i -a source/ dest | jc --rsync-s - or +or $ cat rsync-backup.log | jc --rsync-s diff --git a/docs/parsers/sfdisk.md b/docs/parsers/sfdisk.md index ad6930f97..807afe998 100644 --- a/docs/parsers/sfdisk.md +++ b/docs/parsers/sfdisk.md @@ -18,7 +18,7 @@ Usage (cli): # sfdisk -l | jc --sfdisk - or +or # jc sfdisk -l diff --git a/docs/parsers/ss.md b/docs/parsers/ss.md index 7172fe173..525dbaf61 100644 --- a/docs/parsers/ss.md +++ b/docs/parsers/ss.md @@ -12,7 +12,7 @@ Usage (cli): $ ss | jc --ss - or +or $ jc ss diff --git a/docs/parsers/stat.md b/docs/parsers/stat.md index 95ab2a28d..9cad94a0f 100644 --- a/docs/parsers/stat.md +++ b/docs/parsers/stat.md @@ -15,7 +15,7 @@ Usage (cli): $ stat * | jc --stat - or +or $ jc stat * diff --git a/docs/parsers/sysctl.md b/docs/parsers/sysctl.md index 1481e0b1d..06c998662 100644 --- a/docs/parsers/sysctl.md +++ b/docs/parsers/sysctl.md @@ -14,7 +14,7 @@ Usage (cli): $ sysctl -a | jc --sysctl - or +or $ jc sysctl -a diff --git a/docs/parsers/syslog_bsd.md b/docs/parsers/syslog_bsd.md index 0512b4709..a3f9ecb02 100644 --- a/docs/parsers/syslog_bsd.md +++ b/docs/parsers/syslog_bsd.md @@ -11,7 +11,7 @@ Usage (cli): $ syslogstring | jc --syslog-bsd - or +or $ jc syslog-3164 diff --git a/docs/parsers/systemctl.md b/docs/parsers/systemctl.md index b9e2eb4d2..27ee430bb 100644 --- a/docs/parsers/systemctl.md +++ b/docs/parsers/systemctl.md @@ -9,7 +9,7 @@ Usage (cli): $ systemctl | jc --systemctl - or +or $ jc systemctl diff --git a/docs/parsers/systemctl_lj.md b/docs/parsers/systemctl_lj.md index cc3b26812..77973b1d2 100644 --- a/docs/parsers/systemctl_lj.md +++ b/docs/parsers/systemctl_lj.md @@ -9,7 +9,7 @@ Usage (cli): $ systemctl list-jobs | jc --systemctl-lj - or +or $ jc systemctl list-jobs diff --git a/docs/parsers/systemctl_ls.md b/docs/parsers/systemctl_ls.md index 8de6787e6..28e10cb05 100644 --- a/docs/parsers/systemctl_ls.md +++ b/docs/parsers/systemctl_ls.md @@ -10,7 +10,7 @@ Usage (cli): $ systemctl list-sockets | jc --systemctl-ls - or +or $ jc systemctl list-sockets diff --git a/docs/parsers/systemctl_luf.md b/docs/parsers/systemctl_luf.md index c75055165..7968ef7a3 100644 --- a/docs/parsers/systemctl_luf.md +++ b/docs/parsers/systemctl_luf.md @@ -10,7 +10,7 @@ Usage (cli): $ systemctl list-unit-files | jc --systemctl-luf - or +or $ jc systemctl list-unit-files diff --git a/docs/parsers/timedatectl.md b/docs/parsers/timedatectl.md index ea00c9093..080767e84 100644 --- a/docs/parsers/timedatectl.md +++ b/docs/parsers/timedatectl.md @@ -12,7 +12,7 @@ Usage (cli): $ timedatectl | jc --timedatectl - or +or $ jc timedatectl diff --git a/docs/parsers/top.md b/docs/parsers/top.md index e1738cec4..682ac5ccc 100644 --- a/docs/parsers/top.md +++ b/docs/parsers/top.md @@ -16,7 +16,7 @@ Usage (cli): $ top -b -n 3 | jc --top - or +or $ jc top -b -n 3 diff --git a/docs/parsers/tracepath.md b/docs/parsers/tracepath.md index c394c4b57..a5e2ec922 100644 --- a/docs/parsers/tracepath.md +++ b/docs/parsers/tracepath.md @@ -11,7 +11,7 @@ Usage (cli): $ tracepath 1.2.3.4 | jc --tracepath - or +or $ jc tracepath 1.2.3.4 diff --git a/docs/parsers/traceroute.md b/docs/parsers/traceroute.md index 7b22ebe66..1a0f73472 100644 --- a/docs/parsers/traceroute.md +++ b/docs/parsers/traceroute.md @@ -18,7 +18,7 @@ Usage (cli): $ traceroute 1.2.3.4 | jc --traceroute - or +or $ jc traceroute 1.2.3.4 diff --git a/docs/parsers/ufw.md b/docs/parsers/ufw.md index 24875ecec..f1fb094e2 100644 --- a/docs/parsers/ufw.md +++ b/docs/parsers/ufw.md @@ -9,7 +9,7 @@ Usage (cli): $ ufw status | jc --ufw - or +or $ jc ufw status diff --git a/docs/parsers/ufw_appinfo.md b/docs/parsers/ufw_appinfo.md index 2e11e3a41..24c3848dd 100644 --- a/docs/parsers/ufw_appinfo.md +++ b/docs/parsers/ufw_appinfo.md @@ -17,7 +17,7 @@ Usage (cli): $ ufw app info OpenSSH | jc --ufw-appinfo - or +or $ jc ufw app info OpenSSH diff --git a/docs/parsers/uname.md b/docs/parsers/uname.md index c01eeb591..6ee82c29d 100644 --- a/docs/parsers/uname.md +++ b/docs/parsers/uname.md @@ -11,7 +11,7 @@ Usage (cli): $ uname -a | jc --uname - or +or $ jc uname -a diff --git a/docs/parsers/update_alt_gs.md b/docs/parsers/update_alt_gs.md index 61496af2f..0e0b9175b 100644 --- a/docs/parsers/update_alt_gs.md +++ b/docs/parsers/update_alt_gs.md @@ -9,7 +9,7 @@ Usage (cli): $ update-alternatives --get-selections | jc --update-alt-gs - or +or $ jc update-alternatives --get-selections diff --git a/docs/parsers/update_alt_q.md b/docs/parsers/update_alt_q.md index 2334cf146..f972cd341 100644 --- a/docs/parsers/update_alt_q.md +++ b/docs/parsers/update_alt_q.md @@ -9,7 +9,7 @@ Usage (cli): $ update-alternatives --query | jc --update-alt-q - or +or $ jc update-alternatives --query diff --git a/docs/parsers/upower.md b/docs/parsers/upower.md index ee3ebd916..dfada28fe 100644 --- a/docs/parsers/upower.md +++ b/docs/parsers/upower.md @@ -15,7 +15,7 @@ Usage (cli): $ upower -d | jc --upower - or +or $ jc upower -d diff --git a/docs/parsers/uptime.md b/docs/parsers/uptime.md index 0fa5d82bc..6e22c7118 100644 --- a/docs/parsers/uptime.md +++ b/docs/parsers/uptime.md @@ -9,7 +9,7 @@ Usage (cli): $ uptime | jc --uptime - or +or $ jc uptime diff --git a/docs/parsers/vmstat.md b/docs/parsers/vmstat.md index f02522a43..9cb361b18 100644 --- a/docs/parsers/vmstat.md +++ b/docs/parsers/vmstat.md @@ -17,7 +17,7 @@ Usage (cli): $ vmstat | jc --vmstat - or +or $ jc vmstat diff --git a/docs/parsers/w.md b/docs/parsers/w.md index d17fd13b1..ef7acd065 100644 --- a/docs/parsers/w.md +++ b/docs/parsers/w.md @@ -9,7 +9,7 @@ Usage (cli): $ w | jc --w - or +or $ jc w diff --git a/docs/parsers/wc.md b/docs/parsers/wc.md index ca1d05a63..1fee148b5 100644 --- a/docs/parsers/wc.md +++ b/docs/parsers/wc.md @@ -9,7 +9,7 @@ Usage (cli): $ wc file.txt | jc --wc - or +or $ jc wc file.txt diff --git a/docs/parsers/who.md b/docs/parsers/who.md index 034993139..1abd1a0d4 100644 --- a/docs/parsers/who.md +++ b/docs/parsers/who.md @@ -14,7 +14,7 @@ Usage (cli): $ who | jc --who - or +or $ jc who diff --git a/docs/parsers/xrandr.md b/docs/parsers/xrandr.md index 4369a464a..924b5bf53 100644 --- a/docs/parsers/xrandr.md +++ b/docs/parsers/xrandr.md @@ -9,7 +9,7 @@ Usage (cli): $ xrandr | jc --xrandr - or +or $ jc xrandr diff --git a/docs/parsers/zipinfo.md b/docs/parsers/zipinfo.md index 1b6796210..517e549b2 100644 --- a/docs/parsers/zipinfo.md +++ b/docs/parsers/zipinfo.md @@ -11,7 +11,7 @@ Usage (cli): $ zipinfo | jc --zipinfo - or +or $ jc zipinfo diff --git a/jc/parsers/acpi.py b/jc/parsers/acpi.py index 2f5d1f68a..c1f731c00 100644 --- a/jc/parsers/acpi.py +++ b/jc/parsers/acpi.py @@ -4,7 +4,7 @@ $ acpi -V | jc --acpi - or +or $ jc acpi -V diff --git a/jc/parsers/airport.py b/jc/parsers/airport.py index 18f0431e6..b7e7353ef 100644 --- a/jc/parsers/airport.py +++ b/jc/parsers/airport.py @@ -6,7 +6,7 @@ $ airport -I | jc --airport - or +or $ jc airport -I diff --git a/jc/parsers/airport_s.py b/jc/parsers/airport_s.py index 7089f14fe..cb7d1fe11 100644 --- a/jc/parsers/airport_s.py +++ b/jc/parsers/airport_s.py @@ -6,7 +6,7 @@ $ airport -s | jc --airport-s - or +or $ jc airport -s diff --git a/jc/parsers/arp.py b/jc/parsers/arp.py index 0012b891e..e06003fd7 100644 --- a/jc/parsers/arp.py +++ b/jc/parsers/arp.py @@ -6,7 +6,7 @@ $ arp | jc --arp - or +or $ jc arp diff --git a/jc/parsers/blkid.py b/jc/parsers/blkid.py index 6cd36794e..19969740a 100644 --- a/jc/parsers/blkid.py +++ b/jc/parsers/blkid.py @@ -4,7 +4,7 @@ $ blkid | jc --blkid - or +or $ jc blkid diff --git a/jc/parsers/cef.py b/jc/parsers/cef.py index db89f53b7..bee30ebd9 100644 --- a/jc/parsers/cef.py +++ b/jc/parsers/cef.py @@ -175,12 +175,15 @@ def _process(proc_data: List[Dict]) -> List[Dict]: """ # fix escape chars specified in syslog RFC 5424 and CEF spec # https://www.rfc-editor.org/rfc/rfc5424.html#section-6 + # https://www.microfocus.com/documentation/arcsight/arcsight-smartconnectors-8.3/cef-implementation-standard/Content/CEF/Chapter%201%20What%20is%20CEF.htm?tocpath=_____2#_Toc494359738 escape_map = { r'\\': '\\', - r'\"': r'"', - r'\]': r']', - r'\|': r'|', - r'\=': r'=', + r'\"': '"', + r'\]': ']', + r'\|': '|', + r'\=': '=', + r'\%': '%', + r'\#': '#', r'\n': '\n', r'\r': '\r' } diff --git a/jc/parsers/chage.py b/jc/parsers/chage.py index 1d56908c3..a2ef099f6 100644 --- a/jc/parsers/chage.py +++ b/jc/parsers/chage.py @@ -6,7 +6,7 @@ $ chage -l johndoe | jc --chage - or +or $ jc chage -l johndoe diff --git a/jc/parsers/cksum.py b/jc/parsers/cksum.py index 4368853cd..87428f928 100644 --- a/jc/parsers/cksum.py +++ b/jc/parsers/cksum.py @@ -8,7 +8,7 @@ $ cksum file.txt | jc --cksum - or +or $ jc cksum file.txt diff --git a/jc/parsers/crontab.py b/jc/parsers/crontab.py index 314fa87c6..9f19026fc 100644 --- a/jc/parsers/crontab.py +++ b/jc/parsers/crontab.py @@ -7,7 +7,7 @@ $ crontab -l | jc --crontab - or +or $ jc crontab -l diff --git a/jc/parsers/date.py b/jc/parsers/date.py index 4b6d94300..237ead6dd 100644 --- a/jc/parsers/date.py +++ b/jc/parsers/date.py @@ -10,7 +10,7 @@ $ date | jc --date - or +or $ jc date diff --git a/jc/parsers/df.py b/jc/parsers/df.py index 16c1d2886..e077a1e29 100644 --- a/jc/parsers/df.py +++ b/jc/parsers/df.py @@ -4,7 +4,7 @@ $ df | jc --df - or +or $ jc df diff --git a/jc/parsers/dig.py b/jc/parsers/dig.py index fcb24b685..a093e84b6 100644 --- a/jc/parsers/dig.py +++ b/jc/parsers/dig.py @@ -15,7 +15,7 @@ $ dig example.com | jc --dig - or +or $ jc dig example.com diff --git a/jc/parsers/dmidecode.py b/jc/parsers/dmidecode.py index e2f23ea1c..3b7e180ef 100644 --- a/jc/parsers/dmidecode.py +++ b/jc/parsers/dmidecode.py @@ -4,7 +4,7 @@ $ dmidecode | jc --dmidecode - or +or $ jc dmidecode diff --git a/jc/parsers/dpkg_l.py b/jc/parsers/dpkg_l.py index 0fccc4221..c82ee28aa 100644 --- a/jc/parsers/dpkg_l.py +++ b/jc/parsers/dpkg_l.py @@ -9,7 +9,7 @@ $ dpkg -l | jc --dpkg-l - or +or $ jc dpkg -l diff --git a/jc/parsers/du.py b/jc/parsers/du.py index fa13c1574..ef9d68c9d 100644 --- a/jc/parsers/du.py +++ b/jc/parsers/du.py @@ -4,7 +4,7 @@ $ du | jc --du - or +or $ jc du diff --git a/jc/parsers/env.py b/jc/parsers/env.py index 2dae81644..667a4ffc3 100644 --- a/jc/parsers/env.py +++ b/jc/parsers/env.py @@ -9,7 +9,7 @@ $ env | jc --env - or +or $ jc env diff --git a/jc/parsers/file.py b/jc/parsers/file.py index 4a8adc0d9..10789858d 100644 --- a/jc/parsers/file.py +++ b/jc/parsers/file.py @@ -4,7 +4,7 @@ $ file * | jc --file - or +or $ jc file * diff --git a/jc/parsers/finger.py b/jc/parsers/finger.py index 69c95c064..52f37381f 100644 --- a/jc/parsers/finger.py +++ b/jc/parsers/finger.py @@ -6,7 +6,7 @@ $ finger | jc --finger - or +or $ jc finger diff --git a/jc/parsers/foo.py b/jc/parsers/foo.py index 026775498..3b66980bb 100644 --- a/jc/parsers/foo.py +++ b/jc/parsers/foo.py @@ -6,7 +6,7 @@ $ foo | jc --foo - or +or $ jc foo diff --git a/jc/parsers/free.py b/jc/parsers/free.py index 4b460d321..ec6e58440 100644 --- a/jc/parsers/free.py +++ b/jc/parsers/free.py @@ -4,7 +4,7 @@ $ free | jc --free - or +or $ jc free diff --git a/jc/parsers/git_log.py b/jc/parsers/git_log.py index 5fcc49cbb..088b813e7 100644 --- a/jc/parsers/git_log.py +++ b/jc/parsers/git_log.py @@ -21,7 +21,7 @@ $ git log | jc --git-log - or +or $ jc git log diff --git a/jc/parsers/gpg.py b/jc/parsers/gpg.py index 684a540fb..dff1495a1 100644 --- a/jc/parsers/gpg.py +++ b/jc/parsers/gpg.py @@ -4,7 +4,7 @@ $ gpg --with-colons --show-keys file.gpg | jc --gpg - or +or $ jc gpg --with-colons --show-keys file.gpg diff --git a/jc/parsers/hashsum.py b/jc/parsers/hashsum.py index 34bb43feb..512016a09 100644 --- a/jc/parsers/hashsum.py +++ b/jc/parsers/hashsum.py @@ -14,7 +14,7 @@ $ md5sum file.txt | jc --hashsum - or +or $ jc md5sum file.txt diff --git a/jc/parsers/hciconfig.py b/jc/parsers/hciconfig.py index 2e7d13a33..e21f2b68c 100644 --- a/jc/parsers/hciconfig.py +++ b/jc/parsers/hciconfig.py @@ -4,7 +4,7 @@ $ hciconfig | jc --hciconfig - or +or $ jc hciconfig diff --git a/jc/parsers/id.py b/jc/parsers/id.py index b1e478256..897ba4940 100644 --- a/jc/parsers/id.py +++ b/jc/parsers/id.py @@ -4,7 +4,7 @@ $ id | jc --id - or +or $ jc id diff --git a/jc/parsers/ifconfig.py b/jc/parsers/ifconfig.py index 7c0d1b67f..c1db228ce 100644 --- a/jc/parsers/ifconfig.py +++ b/jc/parsers/ifconfig.py @@ -6,7 +6,7 @@ $ ifconfig | jc --ifconfig - or +or $ jc ifconfig diff --git a/jc/parsers/iostat.py b/jc/parsers/iostat.py index 66982e06b..4bd01f3b6 100644 --- a/jc/parsers/iostat.py +++ b/jc/parsers/iostat.py @@ -6,7 +6,7 @@ $ iostat | jc --iostat - or +or $ jc iostat diff --git a/jc/parsers/iptables.py b/jc/parsers/iptables.py index 302bb3ff5..c33f6917f 100644 --- a/jc/parsers/iptables.py +++ b/jc/parsers/iptables.py @@ -6,7 +6,7 @@ $ sudo iptables -L -t nat | jc --iptables - or +or $ jc iptables -L -t nat diff --git a/jc/parsers/iw_scan.py b/jc/parsers/iw_scan.py index eea984e85..ea453ef04 100644 --- a/jc/parsers/iw_scan.py +++ b/jc/parsers/iw_scan.py @@ -7,7 +7,7 @@ $ iw dev wlan0 scan | jc --iw-scan - or +or $ jc iw dev wlan0 scan diff --git a/jc/parsers/jar_manifest.py b/jc/parsers/jar_manifest.py index 0b7a320e3..4c930fbf3 100644 --- a/jc/parsers/jar_manifest.py +++ b/jc/parsers/jar_manifest.py @@ -1,4 +1,4 @@ -"""jc - JSON Convert `MANIFEST.MF` file parser +"""jc - JSON Convert Java `MANIFEST.MF` file parser Usage (cli): @@ -79,7 +79,7 @@ class info(): """Provides parser metadata (version, author, etc.)""" version = '0.01' - description = 'MANIFEST.MF file parser' + description = 'Java MANIFEST.MF file parser' author = 'Matt J' author_email = 'https://github.com/listuser' compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd'] diff --git a/jc/parsers/last.py b/jc/parsers/last.py index 74948a36f..d3040553a 100644 --- a/jc/parsers/last.py +++ b/jc/parsers/last.py @@ -10,7 +10,7 @@ $ last | jc --last - or +or $ jc last diff --git a/jc/parsers/ls.py b/jc/parsers/ls.py index 3c919e786..8dad76d39 100644 --- a/jc/parsers/ls.py +++ b/jc/parsers/ls.py @@ -21,7 +21,7 @@ $ ls | jc --ls - or +or $ jc ls diff --git a/jc/parsers/lsblk.py b/jc/parsers/lsblk.py index 95365c484..ecf28ab73 100644 --- a/jc/parsers/lsblk.py +++ b/jc/parsers/lsblk.py @@ -4,7 +4,7 @@ $ lsblk | jc --lsblk - or +or $ jc lsblk diff --git a/jc/parsers/lsmod.py b/jc/parsers/lsmod.py index 594c9fd68..035108f16 100644 --- a/jc/parsers/lsmod.py +++ b/jc/parsers/lsmod.py @@ -4,7 +4,7 @@ $ lsmod | jc --lsmod - or +or $ jc lsmod diff --git a/jc/parsers/lsof.py b/jc/parsers/lsof.py index 9f48a7c7b..ef14cd561 100644 --- a/jc/parsers/lsof.py +++ b/jc/parsers/lsof.py @@ -4,7 +4,7 @@ $ lsof | jc --lsof - or +or $ jc lsof diff --git a/jc/parsers/lsusb.py b/jc/parsers/lsusb.py index 58bd6072c..490d5429b 100644 --- a/jc/parsers/lsusb.py +++ b/jc/parsers/lsusb.py @@ -6,7 +6,7 @@ $ lsusb -v | jc --lsusb - or +or $ jc lsusb -v diff --git a/jc/parsers/mount.py b/jc/parsers/mount.py index 048ab9b16..cbddf4d24 100644 --- a/jc/parsers/mount.py +++ b/jc/parsers/mount.py @@ -4,7 +4,7 @@ $ mount | jc --mount - or +or $ jc mount diff --git a/jc/parsers/mpstat.py b/jc/parsers/mpstat.py index 368a64c54..6467a36ca 100644 --- a/jc/parsers/mpstat.py +++ b/jc/parsers/mpstat.py @@ -6,7 +6,7 @@ $ mpstat | jc --mpstat - or +or $ jc mpstat diff --git a/jc/parsers/netstat.py b/jc/parsers/netstat.py index 4e30de778..a74161525 100644 --- a/jc/parsers/netstat.py +++ b/jc/parsers/netstat.py @@ -9,7 +9,7 @@ $ netstat | jc --netstat - or +or $ jc netstat diff --git a/jc/parsers/nmcli.py b/jc/parsers/nmcli.py index 8ac993da0..c9a78f7a0 100644 --- a/jc/parsers/nmcli.py +++ b/jc/parsers/nmcli.py @@ -13,7 +13,7 @@ $ nmcli device show lo | jc --nmcli - or +or $ jc nmcli device show lo diff --git a/jc/parsers/ntpq.py b/jc/parsers/ntpq.py index e48662430..7e0b79f79 100644 --- a/jc/parsers/ntpq.py +++ b/jc/parsers/ntpq.py @@ -4,7 +4,7 @@ $ ntpq -p | jc --ntpq - or +or $ jc ntpq -p diff --git a/jc/parsers/pidstat.py b/jc/parsers/pidstat.py index 42e391cd2..ee031eae9 100644 --- a/jc/parsers/pidstat.py +++ b/jc/parsers/pidstat.py @@ -7,7 +7,7 @@ $ pidstat -h | jc --pidstat - or +or $ jc pidstat -h diff --git a/jc/parsers/ping.py b/jc/parsers/ping.py index 4d7897286..3ca5e51b0 100644 --- a/jc/parsers/ping.py +++ b/jc/parsers/ping.py @@ -9,7 +9,7 @@ $ ping -c 3 1.2.3.4 | jc --ping - or +or $ jc ping -c 3 1.2.3.4 diff --git a/jc/parsers/pip_list.py b/jc/parsers/pip_list.py index 416c7fb09..ebaf8f0b5 100644 --- a/jc/parsers/pip_list.py +++ b/jc/parsers/pip_list.py @@ -4,7 +4,7 @@ $ pip list | jc --pip-list - or +or $ jc pip list diff --git a/jc/parsers/pip_show.py b/jc/parsers/pip_show.py index 732c46ede..4510e92ef 100644 --- a/jc/parsers/pip_show.py +++ b/jc/parsers/pip_show.py @@ -4,7 +4,7 @@ $ pip show | jc --pip-show - or +or $ jc pip show diff --git a/jc/parsers/postconf.py b/jc/parsers/postconf.py index 0983492b8..bfdf999b5 100644 --- a/jc/parsers/postconf.py +++ b/jc/parsers/postconf.py @@ -4,7 +4,7 @@ $ postconf -M | jc --postconf - or +or $ jc postconf -M diff --git a/jc/parsers/ps.py b/jc/parsers/ps.py index 617200324..b7fc7bc3e 100644 --- a/jc/parsers/ps.py +++ b/jc/parsers/ps.py @@ -8,7 +8,7 @@ $ ps | jc --ps - or +or $ jc ps diff --git a/jc/parsers/route.py b/jc/parsers/route.py index 2e530c514..4f7bdfdd6 100644 --- a/jc/parsers/route.py +++ b/jc/parsers/route.py @@ -4,7 +4,7 @@ $ route | jc --route - or +or $ jc route diff --git a/jc/parsers/rpm_qi.py b/jc/parsers/rpm_qi.py index f0061a0dc..21b0c3bd9 100644 --- a/jc/parsers/rpm_qi.py +++ b/jc/parsers/rpm_qi.py @@ -12,7 +12,7 @@ $ rpm -qia | jc --rpm-qi - or +or $ jc rpm -qia diff --git a/jc/parsers/rsync.py b/jc/parsers/rsync.py index c18e487c3..55ae55998 100644 --- a/jc/parsers/rsync.py +++ b/jc/parsers/rsync.py @@ -8,11 +8,11 @@ $ rsync -i -a source/ dest | jc --rsync - or +or $ jc rsync -i -a source/ dest - or +or $ cat rsync-backup.log | jc --rsync diff --git a/jc/parsers/rsync_s.py b/jc/parsers/rsync_s.py index a8ad399bd..45d6edf6b 100644 --- a/jc/parsers/rsync_s.py +++ b/jc/parsers/rsync_s.py @@ -11,7 +11,7 @@ $ rsync -i -a source/ dest | jc --rsync-s - or +or $ cat rsync-backup.log | jc --rsync-s diff --git a/jc/parsers/sfdisk.py b/jc/parsers/sfdisk.py index 93d261cb3..440eb5ac6 100644 --- a/jc/parsers/sfdisk.py +++ b/jc/parsers/sfdisk.py @@ -13,7 +13,7 @@ # sfdisk -l | jc --sfdisk - or +or # jc sfdisk -l diff --git a/jc/parsers/ss.py b/jc/parsers/ss.py index 9e53dd7df..a4564dcd8 100644 --- a/jc/parsers/ss.py +++ b/jc/parsers/ss.py @@ -7,7 +7,7 @@ $ ss | jc --ss - or +or $ jc ss diff --git a/jc/parsers/stat.py b/jc/parsers/stat.py index e49fc047c..706610a78 100644 --- a/jc/parsers/stat.py +++ b/jc/parsers/stat.py @@ -10,7 +10,7 @@ $ stat * | jc --stat - or +or $ jc stat * diff --git a/jc/parsers/sysctl.py b/jc/parsers/sysctl.py index 36ac93c07..22f314d8c 100644 --- a/jc/parsers/sysctl.py +++ b/jc/parsers/sysctl.py @@ -9,7 +9,7 @@ $ sysctl -a | jc --sysctl - or +or $ jc sysctl -a diff --git a/jc/parsers/syslog_bsd.py b/jc/parsers/syslog_bsd.py index 65b264ed0..32ad55a10 100644 --- a/jc/parsers/syslog_bsd.py +++ b/jc/parsers/syslog_bsd.py @@ -6,7 +6,7 @@ $ syslogstring | jc --syslog-bsd - or +or $ jc syslog-3164 diff --git a/jc/parsers/systemctl.py b/jc/parsers/systemctl.py index 71825192b..5e3265fb9 100644 --- a/jc/parsers/systemctl.py +++ b/jc/parsers/systemctl.py @@ -4,7 +4,7 @@ $ systemctl | jc --systemctl - or +or $ jc systemctl diff --git a/jc/parsers/systemctl_lj.py b/jc/parsers/systemctl_lj.py index 01e778f47..531b89dea 100644 --- a/jc/parsers/systemctl_lj.py +++ b/jc/parsers/systemctl_lj.py @@ -4,7 +4,7 @@ $ systemctl list-jobs | jc --systemctl-lj - or +or $ jc systemctl list-jobs diff --git a/jc/parsers/systemctl_ls.py b/jc/parsers/systemctl_ls.py index 9a8a9527b..774ce6c9b 100644 --- a/jc/parsers/systemctl_ls.py +++ b/jc/parsers/systemctl_ls.py @@ -5,7 +5,7 @@ $ systemctl list-sockets | jc --systemctl-ls - or +or $ jc systemctl list-sockets diff --git a/jc/parsers/systemctl_luf.py b/jc/parsers/systemctl_luf.py index 5ed9526ff..da9059cc1 100644 --- a/jc/parsers/systemctl_luf.py +++ b/jc/parsers/systemctl_luf.py @@ -5,7 +5,7 @@ $ systemctl list-unit-files | jc --systemctl-luf - or +or $ jc systemctl list-unit-files diff --git a/jc/parsers/timedatectl.py b/jc/parsers/timedatectl.py index 1944b2ced..d5ecafaf1 100644 --- a/jc/parsers/timedatectl.py +++ b/jc/parsers/timedatectl.py @@ -7,7 +7,7 @@ $ timedatectl | jc --timedatectl - or +or $ jc timedatectl diff --git a/jc/parsers/top.py b/jc/parsers/top.py index c42e68d07..ed6a7d233 100644 --- a/jc/parsers/top.py +++ b/jc/parsers/top.py @@ -11,7 +11,7 @@ $ top -b -n 3 | jc --top - or +or $ jc top -b -n 3 diff --git a/jc/parsers/tracepath.py b/jc/parsers/tracepath.py index dbf9f69f0..25ca91bc4 100644 --- a/jc/parsers/tracepath.py +++ b/jc/parsers/tracepath.py @@ -6,7 +6,7 @@ $ tracepath 1.2.3.4 | jc --tracepath - or +or $ jc tracepath 1.2.3.4 diff --git a/jc/parsers/traceroute.py b/jc/parsers/traceroute.py index e6d423aad..937720123 100644 --- a/jc/parsers/traceroute.py +++ b/jc/parsers/traceroute.py @@ -13,7 +13,7 @@ $ traceroute 1.2.3.4 | jc --traceroute - or +or $ jc traceroute 1.2.3.4 diff --git a/jc/parsers/ufw.py b/jc/parsers/ufw.py index 534163531..f91f06922 100644 --- a/jc/parsers/ufw.py +++ b/jc/parsers/ufw.py @@ -4,7 +4,7 @@ $ ufw status | jc --ufw - or +or $ jc ufw status diff --git a/jc/parsers/ufw_appinfo.py b/jc/parsers/ufw_appinfo.py index beb4cd047..7117c474c 100644 --- a/jc/parsers/ufw_appinfo.py +++ b/jc/parsers/ufw_appinfo.py @@ -12,7 +12,7 @@ $ ufw app info OpenSSH | jc --ufw-appinfo - or +or $ jc ufw app info OpenSSH diff --git a/jc/parsers/uname.py b/jc/parsers/uname.py index 7be864829..4d3f83562 100644 --- a/jc/parsers/uname.py +++ b/jc/parsers/uname.py @@ -6,7 +6,7 @@ $ uname -a | jc --uname - or +or $ jc uname -a diff --git a/jc/parsers/update_alt_gs.py b/jc/parsers/update_alt_gs.py index 13ff4c5e2..6d0f0eec4 100644 --- a/jc/parsers/update_alt_gs.py +++ b/jc/parsers/update_alt_gs.py @@ -4,7 +4,7 @@ $ update-alternatives --get-selections | jc --update-alt-gs - or +or $ jc update-alternatives --get-selections diff --git a/jc/parsers/update_alt_q.py b/jc/parsers/update_alt_q.py index a785c19cb..14c856680 100644 --- a/jc/parsers/update_alt_q.py +++ b/jc/parsers/update_alt_q.py @@ -4,7 +4,7 @@ $ update-alternatives --query | jc --update-alt-q - or +or $ jc update-alternatives --query diff --git a/jc/parsers/upower.py b/jc/parsers/upower.py index e0621b7b8..ea1da1225 100644 --- a/jc/parsers/upower.py +++ b/jc/parsers/upower.py @@ -10,7 +10,7 @@ $ upower -d | jc --upower - or +or $ jc upower -d diff --git a/jc/parsers/uptime.py b/jc/parsers/uptime.py index 8a596e72f..3ae1e8a7e 100644 --- a/jc/parsers/uptime.py +++ b/jc/parsers/uptime.py @@ -4,7 +4,7 @@ $ uptime | jc --uptime - or +or $ jc uptime diff --git a/jc/parsers/vmstat.py b/jc/parsers/vmstat.py index cf7994663..6112b4d71 100644 --- a/jc/parsers/vmstat.py +++ b/jc/parsers/vmstat.py @@ -12,7 +12,7 @@ $ vmstat | jc --vmstat - or +or $ jc vmstat diff --git a/jc/parsers/w.py b/jc/parsers/w.py index 395f4d7d2..6dd7f9f4d 100644 --- a/jc/parsers/w.py +++ b/jc/parsers/w.py @@ -4,7 +4,7 @@ $ w | jc --w - or +or $ jc w diff --git a/jc/parsers/wc.py b/jc/parsers/wc.py index 2d171a0da..6ae1ebed0 100644 --- a/jc/parsers/wc.py +++ b/jc/parsers/wc.py @@ -4,7 +4,7 @@ $ wc file.txt | jc --wc - or +or $ jc wc file.txt diff --git a/jc/parsers/who.py b/jc/parsers/who.py index 80d8093b6..67b1af2a6 100644 --- a/jc/parsers/who.py +++ b/jc/parsers/who.py @@ -9,7 +9,7 @@ $ who | jc --who - or +or $ jc who diff --git a/jc/parsers/xrandr.py b/jc/parsers/xrandr.py index a6da4512d..f475f59d9 100644 --- a/jc/parsers/xrandr.py +++ b/jc/parsers/xrandr.py @@ -4,7 +4,7 @@ $ xrandr | jc --xrandr - or +or $ jc xrandr diff --git a/jc/parsers/zipinfo.py b/jc/parsers/zipinfo.py index edb5f825d..953b74637 100644 --- a/jc/parsers/zipinfo.py +++ b/jc/parsers/zipinfo.py @@ -6,7 +6,7 @@ $ zipinfo | jc --zipinfo - or +or $ jc zipinfo diff --git a/man/jc.1 b/man/jc.1 index c12fbc50d..f2269260c 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -250,7 +250,7 @@ ISO 8601 Datetime string parser .TP .B \fB--jar-manifest\fP -MANIFEST.MF file parser +Java MANIFEST.MF file parser .TP .B From 05fd3cf787d150826ae9b90384aa7679a7a8dadf Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Mon, 15 Aug 2022 15:11:54 -0700 Subject: [PATCH 070/116] add mdadm tests --- .../generic/mdadm-examine-raid0-offline.json | 1 + .../generic/mdadm-examine-raid0-ok.json | 1 + .../generic/mdadm-examine-raid1-0-90-ok.json | 1 + .../generic/mdadm-examine-raid1-checking.json | 1 + .../generic/mdadm-examine-raid1-failfast.json | 1 + .../generic/mdadm-examine-raid1-faulty1.json | 1 + .../generic/mdadm-examine-raid1-faulty2.json | 1 + .../mdadm-examine-raid1-moreflags.json | 1 + .../generic/mdadm-examine-raid1-ok.json | 1 + .../mdadm-examine-raid1-replacing.json | 1 + .../generic/mdadm-examine-raid1-resync.json | 1 + .../generic/mdadm-examine-raid1-spare.json | 1 + .../generic/mdadm-examine-raid1-syncing.json | 1 + tests/fixtures/generic/mdadm-examine.json | 1 + .../fixtures/generic/mdadm-query-detail.json | 1 + .../generic/mdadm-query-raid0-ok.json | 1 + .../mdadm-query-raid1-failed-and-flags.json | 1 + .../mdadm-query-raid1-faulty-and-removed.json | 1 + .../mdadm-query-raid1-faulty-with-spare.json | 1 + .../generic/mdadm-query-raid1-faulty.json | 1 + .../generic/mdadm-query-raid1-ok-0-9.json | 1 + .../mdadm-query-raid1-ok-failfast.json | 1 + .../generic/mdadm-query-raid1-ok-spare.json | 1 + .../generic/mdadm-query-raid1-ok.json | 1 + .../mdadm-query-raid1-rebuild-failfast.json | 1 + ...dadm-query-raid1-spare-writem-rebuild.json | 1 + .../generic/mdadm-query-raid1-syncing.json | 1 + tests/test_mdadm.py | 374 ++++++++++++++++++ 28 files changed, 401 insertions(+) create mode 100644 tests/fixtures/generic/mdadm-examine-raid0-offline.json create mode 100644 tests/fixtures/generic/mdadm-examine-raid0-ok.json create mode 100644 tests/fixtures/generic/mdadm-examine-raid1-0-90-ok.json create mode 100644 tests/fixtures/generic/mdadm-examine-raid1-checking.json create mode 100644 tests/fixtures/generic/mdadm-examine-raid1-failfast.json create mode 100644 tests/fixtures/generic/mdadm-examine-raid1-faulty1.json create mode 100644 tests/fixtures/generic/mdadm-examine-raid1-faulty2.json create mode 100644 tests/fixtures/generic/mdadm-examine-raid1-moreflags.json create mode 100644 tests/fixtures/generic/mdadm-examine-raid1-ok.json create mode 100644 tests/fixtures/generic/mdadm-examine-raid1-replacing.json create mode 100644 tests/fixtures/generic/mdadm-examine-raid1-resync.json create mode 100644 tests/fixtures/generic/mdadm-examine-raid1-spare.json create mode 100644 tests/fixtures/generic/mdadm-examine-raid1-syncing.json create mode 100644 tests/fixtures/generic/mdadm-examine.json create mode 100644 tests/fixtures/generic/mdadm-query-detail.json create mode 100644 tests/fixtures/generic/mdadm-query-raid0-ok.json create mode 100644 tests/fixtures/generic/mdadm-query-raid1-failed-and-flags.json create mode 100644 tests/fixtures/generic/mdadm-query-raid1-faulty-and-removed.json create mode 100644 tests/fixtures/generic/mdadm-query-raid1-faulty-with-spare.json create mode 100644 tests/fixtures/generic/mdadm-query-raid1-faulty.json create mode 100644 tests/fixtures/generic/mdadm-query-raid1-ok-0-9.json create mode 100644 tests/fixtures/generic/mdadm-query-raid1-ok-failfast.json create mode 100644 tests/fixtures/generic/mdadm-query-raid1-ok-spare.json create mode 100644 tests/fixtures/generic/mdadm-query-raid1-ok.json create mode 100644 tests/fixtures/generic/mdadm-query-raid1-rebuild-failfast.json create mode 100644 tests/fixtures/generic/mdadm-query-raid1-spare-writem-rebuild.json create mode 100644 tests/fixtures/generic/mdadm-query-raid1-syncing.json create mode 100644 tests/test_mdadm.py diff --git a/tests/fixtures/generic/mdadm-examine-raid0-offline.json b/tests/fixtures/generic/mdadm-examine-raid0-offline.json new file mode 100644 index 000000000..1ad0d50af --- /dev/null +++ b/tests/fixtures/generic/mdadm-examine-raid0-offline.json @@ -0,0 +1 @@ +{"device":"/dev/vda1","magic":"a92b4efc","version":"1.2","feature_map":"0x0","array_uuid":"7e81a856:abb9c1c2:4b71237a:9778cc66","name":"sysrescue:0 (local to host sysrescue)","creation_time":"Sun Aug 7 20:56:52 2022","raid_level":"raid0","raid_devices":2,"avail_dev_size":"200704 sectors (98.00 MiB 102.76 MB)","data_offset":4096,"super_offset":8,"unused_space":"before=4016 sectors, after=0 sectors","state":"clean","device_uuid":"a49d96f1:8e86ca03:5ff499ec:8cdd42bb","update_time":"Sun Aug 7 20:56:52 2022","bad_block_log":"512 entries available at offset 8 sectors","checksum":"eeb6cbe3 - correct","events":0,"chunk_size":512,"device_role":"Active device 0","array_state":"AA ('A' == active, '.' == missing, 'R' == replacing)","avail_dev_size_num":200704,"unused_space_before":4016,"unused_space_after":0,"name_val":"sysrescue:0","checksum_val":"eeb6cbe3","checksum_state":"correct","state_list":["clean"],"array_state_list":["active","active"],"creation_time_epoch":1659931012,"update_time_epoch":1659931012} diff --git a/tests/fixtures/generic/mdadm-examine-raid0-ok.json b/tests/fixtures/generic/mdadm-examine-raid0-ok.json new file mode 100644 index 000000000..1ad0d50af --- /dev/null +++ b/tests/fixtures/generic/mdadm-examine-raid0-ok.json @@ -0,0 +1 @@ +{"device":"/dev/vda1","magic":"a92b4efc","version":"1.2","feature_map":"0x0","array_uuid":"7e81a856:abb9c1c2:4b71237a:9778cc66","name":"sysrescue:0 (local to host sysrescue)","creation_time":"Sun Aug 7 20:56:52 2022","raid_level":"raid0","raid_devices":2,"avail_dev_size":"200704 sectors (98.00 MiB 102.76 MB)","data_offset":4096,"super_offset":8,"unused_space":"before=4016 sectors, after=0 sectors","state":"clean","device_uuid":"a49d96f1:8e86ca03:5ff499ec:8cdd42bb","update_time":"Sun Aug 7 20:56:52 2022","bad_block_log":"512 entries available at offset 8 sectors","checksum":"eeb6cbe3 - correct","events":0,"chunk_size":512,"device_role":"Active device 0","array_state":"AA ('A' == active, '.' == missing, 'R' == replacing)","avail_dev_size_num":200704,"unused_space_before":4016,"unused_space_after":0,"name_val":"sysrescue:0","checksum_val":"eeb6cbe3","checksum_state":"correct","state_list":["clean"],"array_state_list":["active","active"],"creation_time_epoch":1659931012,"update_time_epoch":1659931012} diff --git a/tests/fixtures/generic/mdadm-examine-raid1-0-90-ok.json b/tests/fixtures/generic/mdadm-examine-raid1-0-90-ok.json new file mode 100644 index 000000000..4f538f22d --- /dev/null +++ b/tests/fixtures/generic/mdadm-examine-raid1-0-90-ok.json @@ -0,0 +1 @@ +{"device":"/dev/vda1","magic":"a92b4efc","version":"0.90.00","uuid":"3f720601:29af5283:96fc04a8:108a4af7 (local to host sysrescue)","creation_time":"Sun Aug 7 21:52:56 2022","raid_level":"raid1","used_dev_size":"102336 (99.94 MiB 104.79 MB)","array_size":"102336 (99.94 MiB 104.79 MB)","raid_devices":2,"total_devices":2,"preferred_minor":0,"update_time":"Sun Aug 7 21:54:35 2022","state":"clean","active_devices":2,"working_devices":2,"failed_devices":0,"spare_devices":0,"checksum":"7fb4bb0e - correct","events":18,"device_table":[{"number":0,"major":254,"minor":1,"state":["active","sync"],"device":"/dev/vda1","raid_device":0},{"number":0,"major":254,"minor":1,"state":["active","sync"],"device":"/dev/vda1","raid_device":0},{"number":11,"major":254,"minor":2,"state":["active","sync"],"device":"/dev/vda2","raid_device":1}],"array_size_num":102336,"used_dev_size_num":102336,"uuid_val":"3f720601:29af5283:96fc04a8:108a4af7","checksum_val":"7fb4bb0e","checksum_state":"correct","state_list":["clean"],"creation_time_epoch":1659934376,"update_time_epoch":1659934475} diff --git a/tests/fixtures/generic/mdadm-examine-raid1-checking.json b/tests/fixtures/generic/mdadm-examine-raid1-checking.json new file mode 100644 index 000000000..9be39db60 --- /dev/null +++ b/tests/fixtures/generic/mdadm-examine-raid1-checking.json @@ -0,0 +1 @@ +{"device":"/dev/md126","version":"1.2","creation_time":"Sun Aug 7 21:15:49 2022","raid_level":"raid1","array_size":"101376 (99.00 MiB 103.81 MB)","used_dev_size":"101376 (99.00 MiB 103.81 MB)","raid_devices":2,"total_devices":2,"persistence":"Superblock is persistent","update_time":"Sun Aug 7 21:23:59 2022","state":"clean, checking","active_devices":2,"working_devices":2,"failed_devices":0,"spare_devices":0,"consistency_policy":"resync","check_status":"21% complete","name":"sysrescue:126 (local to host sysrescue)","uuid":"e054553a:fbccdcfe:0ae80bc8:9379377a","events":18,"device_table":[{"number":0,"major":254,"minor":1,"state":["active","sync"],"device":"/dev/vda1","raid_device":0},{"number":1,"major":254,"minor":2,"state":["active","sync"],"device":"/dev/vda2","raid_device":1}],"array_size_num":101376,"used_dev_size_num":101376,"name_val":"sysrescue:126","uuid_val":"e054553a:fbccdcfe:0ae80bc8:9379377a","state_list":["clean","checking"],"check_status_percent":21,"creation_time_epoch":1659932149,"update_time_epoch":1659932639} diff --git a/tests/fixtures/generic/mdadm-examine-raid1-failfast.json b/tests/fixtures/generic/mdadm-examine-raid1-failfast.json new file mode 100644 index 000000000..d6ed6a454 --- /dev/null +++ b/tests/fixtures/generic/mdadm-examine-raid1-failfast.json @@ -0,0 +1 @@ +{"device":"/dev/vda2","magic":"a92b4efc","version":"1.2","feature_map":"0x0","array_uuid":"e054553a:fbccdcfe:0ae80bc8:9379377a","name":"sysrescue:126 (local to host sysrescue)","creation_time":"Sun Aug 7 21:15:49 2022","raid_level":"raid1","raid_devices":2,"avail_dev_size":"202752 sectors (99.00 MiB 103.81 MB)","array_size":"101376 KiB (99.00 MiB 103.81 MB)","data_offset":2048,"super_offset":8,"unused_space":"before=1968 sectors, after=0 sectors","state":"clean","device_uuid":"140a340b:69d5759c:b87ce67c:6579ce22","flags":"failfast","update_time":"Sun Aug 7 21:37:34 2022","bad_block_log":"512 entries available at offset 16 sectors","checksum":"4bf42c40 - correct","events":51,"device_role":"Active device 1","array_state":"AA ('A' == active, '.' == missing, 'R' == replacing)","array_size_num":101376,"avail_dev_size_num":202752,"unused_space_before":1968,"unused_space_after":0,"name_val":"sysrescue:126","checksum_val":"4bf42c40","checksum_state":"correct","state_list":["clean"],"flag_list":["failfast"],"array_state_list":["active","active"],"creation_time_epoch":1659932149,"update_time_epoch":1659933454} diff --git a/tests/fixtures/generic/mdadm-examine-raid1-faulty1.json b/tests/fixtures/generic/mdadm-examine-raid1-faulty1.json new file mode 100644 index 000000000..b98cccd53 --- /dev/null +++ b/tests/fixtures/generic/mdadm-examine-raid1-faulty1.json @@ -0,0 +1 @@ +{"device":"/dev/vda1","magic":"a92b4efc","version":"1.2","feature_map":"0x0","array_uuid":"e054553a:fbccdcfe:0ae80bc8:9379377a","name":"sysrescue:126 (local to host sysrescue)","creation_time":"Sun Aug 7 21:15:49 2022","raid_level":"raid1","raid_devices":2,"avail_dev_size":"202752 sectors (99.00 MiB 103.81 MB)","array_size":"101376 KiB (99.00 MiB 103.81 MB)","data_offset":2048,"super_offset":8,"unused_space":"before=1968 sectors, after=0 sectors","state":"clean","device_uuid":"1ec6a01d:e96d7f47:59970cf2:f0c056d2","update_time":"Sun Aug 7 21:28:34 2022","bad_block_log":"512 entries available at offset 16 sectors","checksum":"2e18e0bb - correct","events":25,"device_role":"Active device 0","array_state":"A. ('A' == active, '.' == missing, 'R' == replacing)","array_size_num":101376,"avail_dev_size_num":202752,"unused_space_before":1968,"unused_space_after":0,"name_val":"sysrescue:126","checksum_val":"2e18e0bb","checksum_state":"correct","state_list":["clean"],"array_state_list":["active","missing"],"creation_time_epoch":1659932149,"update_time_epoch":1659932914} diff --git a/tests/fixtures/generic/mdadm-examine-raid1-faulty2.json b/tests/fixtures/generic/mdadm-examine-raid1-faulty2.json new file mode 100644 index 000000000..95ac28507 --- /dev/null +++ b/tests/fixtures/generic/mdadm-examine-raid1-faulty2.json @@ -0,0 +1 @@ +{"device":"/dev/vda2","magic":"a92b4efc","version":"1.2","feature_map":"0x0","array_uuid":"e054553a:fbccdcfe:0ae80bc8:9379377a","name":"sysrescue:126 (local to host sysrescue)","creation_time":"Sun Aug 7 21:15:49 2022","raid_level":"raid1","raid_devices":2,"avail_dev_size":"202752 sectors (99.00 MiB 103.81 MB)","array_size":"101376 KiB (99.00 MiB 103.81 MB)","data_offset":2048,"super_offset":8,"unused_space":"before=1968 sectors, after=0 sectors","state":"clean","device_uuid":"66e4da05:9cc62990:0620d976:579c0948","update_time":"Sun Aug 7 21:27:02 2022","bad_block_log":"512 entries available at offset 16 sectors","checksum":"597fbb6b - correct","events":23,"device_role":"Active device 1","array_state":"AA ('A' == active, '.' == missing, 'R' == replacing)","array_size_num":101376,"avail_dev_size_num":202752,"unused_space_before":1968,"unused_space_after":0,"name_val":"sysrescue:126","checksum_val":"597fbb6b","checksum_state":"correct","state_list":["clean"],"array_state_list":["active","active"],"creation_time_epoch":1659932149,"update_time_epoch":1659932822} diff --git a/tests/fixtures/generic/mdadm-examine-raid1-moreflags.json b/tests/fixtures/generic/mdadm-examine-raid1-moreflags.json new file mode 100644 index 000000000..c17470281 --- /dev/null +++ b/tests/fixtures/generic/mdadm-examine-raid1-moreflags.json @@ -0,0 +1 @@ +{"device":"/dev/vda2","magic":"a92b4efc","version":"1.2","feature_map":"0x0","array_uuid":"e054553a:fbccdcfe:0ae80bc8:9379377a","name":"sysrescue:126 (local to host sysrescue)","creation_time":"Sun Aug 7 21:15:49 2022","raid_level":"raid1","raid_devices":2,"avail_dev_size":"202752 sectors (99.00 MiB 103.81 MB)","array_size":"101376 KiB (99.00 MiB 103.81 MB)","data_offset":2048,"super_offset":8,"unused_space":"before=1968 sectors, after=0 sectors","state":"clean","device_uuid":"cc336e3b:67a49dea:480135ed:9e1280cd","flags":"write-mostly failfast","update_time":"Sun Aug 7 21:48:58 2022","bad_block_log":"512 entries available at offset 16 sectors","checksum":"e5554588 - correct","events":78,"device_role":"Active device 1","array_state":"AA ('A' == active, '.' == missing, 'R' == replacing)","array_size_num":101376,"avail_dev_size_num":202752,"unused_space_before":1968,"unused_space_after":0,"name_val":"sysrescue:126","checksum_val":"e5554588","checksum_state":"correct","state_list":["clean"],"flag_list":["write-mostly","failfast"],"array_state_list":["active","active"],"creation_time_epoch":1659932149,"update_time_epoch":1659934138} diff --git a/tests/fixtures/generic/mdadm-examine-raid1-ok.json b/tests/fixtures/generic/mdadm-examine-raid1-ok.json new file mode 100644 index 000000000..3a271f974 --- /dev/null +++ b/tests/fixtures/generic/mdadm-examine-raid1-ok.json @@ -0,0 +1 @@ +{"device":"/dev/vda1","magic":"a92b4efc","version":"1.2","feature_map":"0x0","array_uuid":"e054553a:fbccdcfe:0ae80bc8:9379377a","name":"sysrescue:126 (local to host sysrescue)","creation_time":"Sun Aug 7 21:15:49 2022","raid_level":"raid1","raid_devices":2,"avail_dev_size":"202752 sectors (99.00 MiB 103.81 MB)","array_size":"101376 KiB (99.00 MiB 103.81 MB)","data_offset":2048,"super_offset":8,"unused_space":"before=1968 sectors, after=0 sectors","state":"clean","device_uuid":"1ec6a01d:e96d7f47:59970cf2:f0c056d2","update_time":"Sun Aug 7 21:20:08 2022","bad_block_log":"512 entries available at offset 16 sectors","checksum":"2e1bdeb8 - correct","events":17,"device_role":"Active device 0","array_state":"AA ('A' == active, '.' == missing, 'R' == replacing)","array_size_num":101376,"avail_dev_size_num":202752,"unused_space_before":1968,"unused_space_after":0,"name_val":"sysrescue:126","checksum_val":"2e1bdeb8","checksum_state":"correct","state_list":["clean"],"array_state_list":["active","active"],"creation_time_epoch":1659932149,"update_time_epoch":1659932408} diff --git a/tests/fixtures/generic/mdadm-examine-raid1-replacing.json b/tests/fixtures/generic/mdadm-examine-raid1-replacing.json new file mode 100644 index 000000000..92f496298 --- /dev/null +++ b/tests/fixtures/generic/mdadm-examine-raid1-replacing.json @@ -0,0 +1 @@ +{"device":"/dev/vda2","magic":"a92b4efc","version":"1.2","feature_map":"0x12","array_uuid":"e054553a:fbccdcfe:0ae80bc8:9379377a","name":"sysrescue:126 (local to host sysrescue)","creation_time":"Sun Aug 7 21:15:49 2022","raid_level":"raid1","raid_devices":2,"avail_dev_size":"202752 sectors (99.00 MiB 103.81 MB)","array_size":"101376 KiB (99.00 MiB 103.81 MB)","data_offset":2048,"super_offset":8,"recovery_offset":"115200 sectors","unused_space":"before=1968 sectors, after=0 sectors","state":"clean","device_uuid":"cc336e3b:67a49dea:480135ed:9e1280cd","flags":"write-mostly failfast","update_time":"Sun Aug 7 21:48:15 2022","bad_block_log":"512 entries available at offset 16 sectors","checksum":"e55a076b - correct","events":75,"device_role":"Replacement device 1","array_state":"AR ('A' == active, '.' == missing, 'R' == replacing)","array_size_num":101376,"avail_dev_size_num":202752,"unused_space_before":1968,"unused_space_after":0,"name_val":"sysrescue:126","checksum_val":"e55a076b","checksum_state":"correct","state_list":["clean"],"flag_list":["write-mostly","failfast"],"array_state_list":["active","replacing"],"creation_time_epoch":1659932149,"update_time_epoch":1659934095} diff --git a/tests/fixtures/generic/mdadm-examine-raid1-resync.json b/tests/fixtures/generic/mdadm-examine-raid1-resync.json new file mode 100644 index 000000000..69c9fe0c2 --- /dev/null +++ b/tests/fixtures/generic/mdadm-examine-raid1-resync.json @@ -0,0 +1 @@ +{"device":"/dev/md126","version":"1.2","creation_time":"Sun Aug 7 21:15:49 2022","raid_level":"raid1","array_size":"101376 (99.00 MiB 103.81 MB)","used_dev_size":"101376 (99.00 MiB 103.81 MB)","raid_devices":2,"total_devices":2,"persistence":"Superblock is persistent","update_time":"Sun Aug 7 21:25:52 2022","state":"clean, resyncing","active_devices":2,"working_devices":2,"failed_devices":0,"spare_devices":0,"consistency_policy":"resync","resync_status":"27% complete","name":"sysrescue:126 (local to host sysrescue)","uuid":"e054553a:fbccdcfe:0ae80bc8:9379377a","events":20,"device_table":[{"number":0,"major":254,"minor":1,"state":["active","sync"],"device":"/dev/vda1","raid_device":0},{"number":1,"major":254,"minor":2,"state":["active","sync"],"device":"/dev/vda2","raid_device":1}],"array_size_num":101376,"used_dev_size_num":101376,"name_val":"sysrescue:126","uuid_val":"e054553a:fbccdcfe:0ae80bc8:9379377a","state_list":["clean","resyncing"],"resync_status_percent":27,"creation_time_epoch":1659932149,"update_time_epoch":1659932752} diff --git a/tests/fixtures/generic/mdadm-examine-raid1-spare.json b/tests/fixtures/generic/mdadm-examine-raid1-spare.json new file mode 100644 index 000000000..4a90521ef --- /dev/null +++ b/tests/fixtures/generic/mdadm-examine-raid1-spare.json @@ -0,0 +1 @@ +{"device":"/dev/vda4","magic":"a92b4efc","version":"1.2","feature_map":"0x0","array_uuid":"e054553a:fbccdcfe:0ae80bc8:9379377a","name":"sysrescue:126 (local to host sysrescue)","creation_time":"Sun Aug 7 21:15:49 2022","raid_level":"raid1","raid_devices":2,"avail_dev_size":"202752 sectors (99.00 MiB 103.81 MB)","array_size":"101376 KiB (99.00 MiB 103.81 MB)","data_offset":2048,"super_offset":8,"unused_space":"before=1968 sectors, after=0 sectors","state":"clean","device_uuid":"46b6f457:a3cf3db3:41217976:74a7243c","update_time":"Sun Aug 7 21:39:58 2022","bad_block_log":"512 entries available at offset 16 sectors","checksum":"c265a5d4 - correct","events":52,"device_role":"spare","array_state":"AA ('A' == active, '.' == missing, 'R' == replacing)","array_size_num":101376,"avail_dev_size_num":202752,"unused_space_before":1968,"unused_space_after":0,"name_val":"sysrescue:126","checksum_val":"c265a5d4","checksum_state":"correct","state_list":["clean"],"array_state_list":["active","active"],"creation_time_epoch":1659932149,"update_time_epoch":1659933598} diff --git a/tests/fixtures/generic/mdadm-examine-raid1-syncing.json b/tests/fixtures/generic/mdadm-examine-raid1-syncing.json new file mode 100644 index 000000000..5bd81ed41 --- /dev/null +++ b/tests/fixtures/generic/mdadm-examine-raid1-syncing.json @@ -0,0 +1 @@ +{"device":"/dev/vda1","magic":"a92b4efc","version":"1.2","feature_map":"0x0","array_uuid":"e054553a:fbccdcfe:0ae80bc8:9379377a","name":"sysrescue:126 (local to host sysrescue)","creation_time":"Sun Aug 7 21:15:49 2022","raid_level":"raid1","raid_devices":2,"avail_dev_size":"202752 sectors (99.00 MiB 103.81 MB)","array_size":"101376 KiB (99.00 MiB 103.81 MB)","data_offset":2048,"super_offset":8,"unused_space":"before=1968 sectors, after=0 sectors","state":"active","device_uuid":"1ec6a01d:e96d7f47:59970cf2:f0c056d2","update_time":"Sun Aug 7 21:15:49 2022","bad_block_log":"512 entries available at offset 16 sectors","checksum":"2e1bdda4 - correct","events":0,"device_role":"Active device 0","array_state":"AA ('A' == active, '.' == missing, 'R' == replacing)","array_size_num":101376,"avail_dev_size_num":202752,"unused_space_before":1968,"unused_space_after":0,"name_val":"sysrescue:126","checksum_val":"2e1bdda4","checksum_state":"correct","state_list":["active"],"array_state_list":["active","active"],"creation_time_epoch":1659932149,"update_time_epoch":1659932149} diff --git a/tests/fixtures/generic/mdadm-examine.json b/tests/fixtures/generic/mdadm-examine.json new file mode 100644 index 000000000..520a1b1ad --- /dev/null +++ b/tests/fixtures/generic/mdadm-examine.json @@ -0,0 +1 @@ +{"device":"/dev/sdb1","magic":"a92b4efc","version":"1.1","feature_map":"0x1","array_uuid":"85c5b164:d58a5ada:14f5fe07:d642e843","name":"virttest:0","creation_time":"Tue Apr 13 23:22:16 2010","raid_level":"raid1","raid_devices":2,"avail_dev_size":"11721041656 sectors (5.46 TiB 6.00 TB)","array_size":"5860520828 KiB (5.46 TiB 6.00 TB)","data_offset":264,"super_offset":0,"unused_space":"before=80 sectors, after=0 sectors","state":"clean","device_uuid":"813162e5:2e865efe:02ba5570:7003165c","internal_bitmap":"8 sectors from superblock","update_time":"Tue Jul 26 20:16:31 2022","bad_block_log":"512 entries available at offset 72 sectors","checksum":"f141a577 - correct","events":2193679,"device_role":"Active device 0","array_state":"AA ('A' == active, '.' == missing, 'R' == replacing)","array_size_num":5860520828,"avail_dev_size_num":11721041656,"unused_space_before":80,"unused_space_after":0,"name_val":"virttest:0","checksum_val":"f141a577","checksum_state":"correct","state_list":["clean"],"array_state_list":["active","active"],"creation_time_epoch":1271226136,"update_time_epoch":1658891791} diff --git a/tests/fixtures/generic/mdadm-query-detail.json b/tests/fixtures/generic/mdadm-query-detail.json new file mode 100644 index 000000000..8277054d2 --- /dev/null +++ b/tests/fixtures/generic/mdadm-query-detail.json @@ -0,0 +1 @@ +{"device":"/dev/md0","version":"1.1","creation_time":"Tue Apr 13 23:22:16 2010","raid_level":"raid1","array_size":"5860520828 (5.46 TiB 6.00 TB)","used_dev_size":"5860520828 (5.46 TiB 6.00 TB)","raid_devices":2,"total_devices":2,"persistence":"Superblock is persistent","intent_bitmap":"Internal","update_time":"Tue Jul 26 20:16:31 2022","state":"clean","active_devices":2,"working_devices":2,"failed_devices":0,"spare_devices":0,"consistency_policy":"bitmap","name":"virttest:0","uuid":"85c5b164:d58a5ada:14f5fe07:d642e843","events":2193679,"device_table":[{"number":3,"major":8,"minor":17,"state":["active","sync"],"device":"/dev/sdb1","raid_device":0},{"number":2,"major":8,"minor":33,"state":["active","sync"],"device":"/dev/sdc1","raid_device":1}],"array_size_num":5860520828,"used_dev_size_num":5860520828,"name_val":"virttest:0","uuid_val":"85c5b164:d58a5ada:14f5fe07:d642e843","state_list":["clean"],"creation_time_epoch":1271226136,"update_time_epoch":1658891791} diff --git a/tests/fixtures/generic/mdadm-query-raid0-ok.json b/tests/fixtures/generic/mdadm-query-raid0-ok.json new file mode 100644 index 000000000..d792f119d --- /dev/null +++ b/tests/fixtures/generic/mdadm-query-raid0-ok.json @@ -0,0 +1 @@ +{"device":"/dev/md0","version":"1.2","creation_time":"Sun Aug 7 20:56:52 2022","raid_level":"raid0","array_size":"200704 (196.00 MiB 205.52 MB)","raid_devices":2,"total_devices":2,"persistence":"Superblock is persistent","update_time":"Sun Aug 7 20:56:52 2022","state":"clean","active_devices":2,"working_devices":2,"failed_devices":0,"spare_devices":0,"layout":"-unknown-","chunk_size":512,"consistency_policy":"none","name":"sysrescue:0 (local to host sysrescue)","uuid":"7e81a856:abb9c1c2:4b71237a:9778cc66","events":0,"device_table":[{"number":0,"major":254,"minor":1,"state":["active","sync"],"device":"/dev/vda1","raid_device":0},{"number":1,"major":254,"minor":2,"state":["active","sync"],"device":"/dev/vda2","raid_device":1}],"array_size_num":200704,"name_val":"sysrescue:0","uuid_val":"7e81a856:abb9c1c2:4b71237a:9778cc66","state_list":["clean"],"creation_time_epoch":1659931012,"update_time_epoch":1659931012} diff --git a/tests/fixtures/generic/mdadm-query-raid1-failed-and-flags.json b/tests/fixtures/generic/mdadm-query-raid1-failed-and-flags.json new file mode 100644 index 000000000..3afab5d3d --- /dev/null +++ b/tests/fixtures/generic/mdadm-query-raid1-failed-and-flags.json @@ -0,0 +1 @@ +{"device":"/dev/md127","version":"1.2","creation_time":"Sun Aug 7 21:15:49 2022","raid_level":"raid1","array_size":"101376 (99.00 MiB 103.81 MB)","used_dev_size":"101376 (99.00 MiB 103.81 MB)","raid_devices":2,"total_devices":3,"persistence":"Superblock is persistent","update_time":"Sun Aug 7 21:48:58 2022","state":"clean","active_devices":2,"working_devices":2,"failed_devices":1,"spare_devices":0,"consistency_policy":"resync","name":"sysrescue:126 (local to host sysrescue)","uuid":"e054553a:fbccdcfe:0ae80bc8:9379377a","events":78,"device_table":[{"number":0,"major":254,"minor":1,"state":["active","sync"],"device":"/dev/vda1","raid_device":0},{"number":2,"major":254,"minor":2,"state":["active","sync","writemostly","failfast"],"device":"/dev/vda2","raid_device":1},{"number":3,"major":254,"minor":4,"state":["faulty"],"device":"/dev/vda4","raid_device":null}],"array_size_num":101376,"used_dev_size_num":101376,"name_val":"sysrescue:126","uuid_val":"e054553a:fbccdcfe:0ae80bc8:9379377a","state_list":["clean"],"creation_time_epoch":1659932149,"update_time_epoch":1659934138} diff --git a/tests/fixtures/generic/mdadm-query-raid1-faulty-and-removed.json b/tests/fixtures/generic/mdadm-query-raid1-faulty-and-removed.json new file mode 100644 index 000000000..ce6cedd42 --- /dev/null +++ b/tests/fixtures/generic/mdadm-query-raid1-faulty-and-removed.json @@ -0,0 +1 @@ +{"device":"/dev/md127","version":"1.2","creation_time":"Sun Aug 7 21:15:49 2022","raid_level":"raid1","array_size":"101376 (99.00 MiB 103.81 MB)","used_dev_size":"101376 (99.00 MiB 103.81 MB)","raid_devices":2,"total_devices":1,"persistence":"Superblock is persistent","update_time":"Sun Aug 7 21:32:27 2022","state":"clean, degraded","active_devices":1,"working_devices":1,"failed_devices":0,"spare_devices":0,"consistency_policy":"resync","name":"sysrescue:126 (local to host sysrescue)","uuid":"e054553a:fbccdcfe:0ae80bc8:9379377a","events":31,"device_table":[{"number":0,"major":254,"minor":1,"state":["active","sync"],"device":"/dev/vda1","raid_device":0},{"number":null,"major":0,"minor":0,"state":["removed"],"device":null,"raid_device":1}],"array_size_num":101376,"used_dev_size_num":101376,"name_val":"sysrescue:126","uuid_val":"e054553a:fbccdcfe:0ae80bc8:9379377a","state_list":["clean","degraded"],"creation_time_epoch":1659932149,"update_time_epoch":1659933147} diff --git a/tests/fixtures/generic/mdadm-query-raid1-faulty-with-spare.json b/tests/fixtures/generic/mdadm-query-raid1-faulty-with-spare.json new file mode 100644 index 000000000..b25b224c1 --- /dev/null +++ b/tests/fixtures/generic/mdadm-query-raid1-faulty-with-spare.json @@ -0,0 +1 @@ +{"device":"/dev/md127","version":"1.2","creation_time":"Sun Aug 7 21:15:49 2022","raid_level":"raid1","array_size":"101376 (99.00 MiB 103.81 MB)","used_dev_size":"101376 (99.00 MiB 103.81 MB)","raid_devices":2,"total_devices":3,"persistence":"Superblock is persistent","update_time":"Sun Aug 7 21:43:18 2022","state":"clean, degraded, recovering","active_devices":1,"working_devices":2,"failed_devices":1,"spare_devices":1,"consistency_policy":"resync","rebuild_status":"24% complete","name":"sysrescue:126 (local to host sysrescue)","uuid":"e054553a:fbccdcfe:0ae80bc8:9379377a","events":57,"device_table":[{"number":0,"major":254,"minor":1,"state":["active","sync"],"device":"/dev/vda1","raid_device":0},{"number":3,"major":254,"minor":4,"state":["spare","rebuilding"],"device":"/dev/vda4","raid_device":1},{"number":2,"major":254,"minor":2,"state":["faulty","failfast"],"device":"/dev/vda2","raid_device":null}],"array_size_num":101376,"used_dev_size_num":101376,"name_val":"sysrescue:126","uuid_val":"e054553a:fbccdcfe:0ae80bc8:9379377a","state_list":["clean","degraded","recovering"],"rebuild_status_percent":24,"creation_time_epoch":1659932149,"update_time_epoch":1659933798} diff --git a/tests/fixtures/generic/mdadm-query-raid1-faulty.json b/tests/fixtures/generic/mdadm-query-raid1-faulty.json new file mode 100644 index 000000000..3a2c8e0f2 --- /dev/null +++ b/tests/fixtures/generic/mdadm-query-raid1-faulty.json @@ -0,0 +1 @@ +{"device":"/dev/md126","version":"1.2","creation_time":"Sun Aug 7 21:15:49 2022","raid_level":"raid1","array_size":"101376 (99.00 MiB 103.81 MB)","used_dev_size":"101376 (99.00 MiB 103.81 MB)","raid_devices":2,"total_devices":2,"persistence":"Superblock is persistent","update_time":"Sun Aug 7 21:28:34 2022","state":"clean, degraded","active_devices":1,"working_devices":1,"failed_devices":1,"spare_devices":0,"consistency_policy":"resync","name":"sysrescue:126 (local to host sysrescue)","uuid":"e054553a:fbccdcfe:0ae80bc8:9379377a","events":25,"device_table":[{"number":0,"major":254,"minor":1,"state":["active","sync"],"device":"/dev/vda1","raid_device":0},{"number":null,"major":0,"minor":0,"state":["removed"],"device":null,"raid_device":1},{"number":1,"major":254,"minor":2,"state":["faulty"],"device":"/dev/vda2","raid_device":null}],"array_size_num":101376,"used_dev_size_num":101376,"name_val":"sysrescue:126","uuid_val":"e054553a:fbccdcfe:0ae80bc8:9379377a","state_list":["clean","degraded"],"creation_time_epoch":1659932149,"update_time_epoch":1659932914} diff --git a/tests/fixtures/generic/mdadm-query-raid1-ok-0-9.json b/tests/fixtures/generic/mdadm-query-raid1-ok-0-9.json new file mode 100644 index 000000000..a0e57efcd --- /dev/null +++ b/tests/fixtures/generic/mdadm-query-raid1-ok-0-9.json @@ -0,0 +1 @@ +{"device":"/dev/md0","version":"0.90","creation_time":"Sun Aug 7 21:52:56 2022","raid_level":"raid1","array_size":"102336 (99.94 MiB 104.79 MB)","used_dev_size":"102336 (99.94 MiB 104.79 MB)","raid_devices":2,"total_devices":2,"preferred_minor":0,"persistence":"Superblock is persistent","update_time":"Sun Aug 7 21:54:35 2022","state":"clean","active_devices":2,"working_devices":2,"failed_devices":0,"spare_devices":0,"consistency_policy":"resync","uuid":"3f720601:29af5283:96fc04a8:108a4af7 (local to host sysrescue)","events":0,"device_table":[{"number":0,"major":254,"minor":1,"state":["active","sync"],"device":"/dev/vda1","raid_device":0},{"number":1,"major":254,"minor":2,"state":["active","sync"],"device":"/dev/vda2","raid_device":1}],"array_size_num":102336,"used_dev_size_num":102336,"uuid_val":"3f720601:29af5283:96fc04a8:108a4af7","state_list":["clean"],"creation_time_epoch":1659934376,"update_time_epoch":1659934475} diff --git a/tests/fixtures/generic/mdadm-query-raid1-ok-failfast.json b/tests/fixtures/generic/mdadm-query-raid1-ok-failfast.json new file mode 100644 index 000000000..d741972df --- /dev/null +++ b/tests/fixtures/generic/mdadm-query-raid1-ok-failfast.json @@ -0,0 +1 @@ +{"device":"/dev/md127","version":"1.2","creation_time":"Sun Aug 7 21:15:49 2022","raid_level":"raid1","array_size":"101376 (99.00 MiB 103.81 MB)","used_dev_size":"101376 (99.00 MiB 103.81 MB)","raid_devices":2,"total_devices":2,"persistence":"Superblock is persistent","update_time":"Sun Aug 7 21:37:34 2022","state":"clean","active_devices":2,"working_devices":2,"failed_devices":0,"spare_devices":0,"consistency_policy":"resync","name":"sysrescue:126 (local to host sysrescue)","uuid":"e054553a:fbccdcfe:0ae80bc8:9379377a","events":51,"device_table":[{"number":0,"major":254,"minor":1,"state":["active","sync"],"device":"/dev/vda1","raid_device":0},{"number":2,"major":254,"minor":2,"state":["active","sync","failfast"],"device":"/dev/vda2","raid_device":1}],"array_size_num":101376,"used_dev_size_num":101376,"name_val":"sysrescue:126","uuid_val":"e054553a:fbccdcfe:0ae80bc8:9379377a","state_list":["clean"],"creation_time_epoch":1659932149,"update_time_epoch":1659933454} diff --git a/tests/fixtures/generic/mdadm-query-raid1-ok-spare.json b/tests/fixtures/generic/mdadm-query-raid1-ok-spare.json new file mode 100644 index 000000000..2aa630880 --- /dev/null +++ b/tests/fixtures/generic/mdadm-query-raid1-ok-spare.json @@ -0,0 +1 @@ +{"device":"/dev/md127","version":"1.2","creation_time":"Sun Aug 7 21:15:49 2022","raid_level":"raid1","array_size":"101376 (99.00 MiB 103.81 MB)","used_dev_size":"101376 (99.00 MiB 103.81 MB)","raid_devices":2,"total_devices":3,"persistence":"Superblock is persistent","update_time":"Sun Aug 7 21:39:58 2022","state":"clean","active_devices":2,"working_devices":3,"failed_devices":0,"spare_devices":1,"consistency_policy":"resync","name":"sysrescue:126 (local to host sysrescue)","uuid":"e054553a:fbccdcfe:0ae80bc8:9379377a","events":52,"device_table":[{"number":0,"major":254,"minor":1,"state":["active","sync"],"device":"/dev/vda1","raid_device":0},{"number":2,"major":254,"minor":2,"state":["active","sync","failfast"],"device":"/dev/vda2","raid_device":1},{"number":3,"major":254,"minor":4,"state":["spare"],"device":"/dev/vda4","raid_device":null}],"array_size_num":101376,"used_dev_size_num":101376,"name_val":"sysrescue:126","uuid_val":"e054553a:fbccdcfe:0ae80bc8:9379377a","state_list":["clean"],"creation_time_epoch":1659932149,"update_time_epoch":1659933598} diff --git a/tests/fixtures/generic/mdadm-query-raid1-ok.json b/tests/fixtures/generic/mdadm-query-raid1-ok.json new file mode 100644 index 000000000..4e0173387 --- /dev/null +++ b/tests/fixtures/generic/mdadm-query-raid1-ok.json @@ -0,0 +1 @@ +{"device":"/dev/md126","version":"1.2","creation_time":"Sun Aug 7 21:15:49 2022","raid_level":"raid1","array_size":"101376 (99.00 MiB 103.81 MB)","used_dev_size":"101376 (99.00 MiB 103.81 MB)","raid_devices":2,"total_devices":2,"persistence":"Superblock is persistent","update_time":"Sun Aug 7 21:20:08 2022","state":"clean","active_devices":2,"working_devices":2,"failed_devices":0,"spare_devices":0,"consistency_policy":"resync","name":"sysrescue:126 (local to host sysrescue)","uuid":"e054553a:fbccdcfe:0ae80bc8:9379377a","events":17,"device_table":[{"number":0,"major":254,"minor":1,"state":["active","sync"],"device":"/dev/vda1","raid_device":0},{"number":1,"major":254,"minor":2,"state":["active","sync"],"device":"/dev/vda2","raid_device":1}],"array_size_num":101376,"used_dev_size_num":101376,"name_val":"sysrescue:126","uuid_val":"e054553a:fbccdcfe:0ae80bc8:9379377a","state_list":["clean"],"creation_time_epoch":1659932149,"update_time_epoch":1659932408} diff --git a/tests/fixtures/generic/mdadm-query-raid1-rebuild-failfast.json b/tests/fixtures/generic/mdadm-query-raid1-rebuild-failfast.json new file mode 100644 index 000000000..1dff84061 --- /dev/null +++ b/tests/fixtures/generic/mdadm-query-raid1-rebuild-failfast.json @@ -0,0 +1 @@ +{"device":"/dev/md127","version":"1.2","creation_time":"Sun Aug 7 21:15:49 2022","raid_level":"raid1","array_size":"101376 (99.00 MiB 103.81 MB)","used_dev_size":"101376 (99.00 MiB 103.81 MB)","raid_devices":2,"total_devices":2,"persistence":"Superblock is persistent","update_time":"Sun Aug 7 21:36:17 2022","state":"clean, degraded, recovering","active_devices":1,"working_devices":2,"failed_devices":0,"spare_devices":1,"consistency_policy":"resync","rebuild_status":"20% complete","name":"sysrescue:126 (local to host sysrescue)","uuid":"e054553a:fbccdcfe:0ae80bc8:9379377a","events":37,"device_table":[{"number":0,"major":254,"minor":1,"state":["active","sync"],"device":"/dev/vda1","raid_device":0},{"number":2,"major":254,"minor":2,"state":["failfast","spare","rebuilding"],"device":"/dev/vda2","raid_device":1}],"array_size_num":101376,"used_dev_size_num":101376,"name_val":"sysrescue:126","uuid_val":"e054553a:fbccdcfe:0ae80bc8:9379377a","state_list":["clean","degraded","recovering"],"rebuild_status_percent":20,"creation_time_epoch":1659932149,"update_time_epoch":1659933377} diff --git a/tests/fixtures/generic/mdadm-query-raid1-spare-writem-rebuild.json b/tests/fixtures/generic/mdadm-query-raid1-spare-writem-rebuild.json new file mode 100644 index 000000000..2549d918f --- /dev/null +++ b/tests/fixtures/generic/mdadm-query-raid1-spare-writem-rebuild.json @@ -0,0 +1 @@ +{"device":"/dev/md127","version":"1.2","creation_time":"Sun Aug 7 21:15:49 2022","raid_level":"raid1","array_size":"101376 (99.00 MiB 103.81 MB)","used_dev_size":"101376 (99.00 MiB 103.81 MB)","raid_devices":2,"total_devices":3,"persistence":"Superblock is persistent","update_time":"Sun Aug 7 21:47:42 2022","state":"clean, recovering","active_devices":2,"working_devices":3,"failed_devices":0,"spare_devices":1,"consistency_policy":"resync","rebuild_status":"30% complete","name":"sysrescue:126 (local to host sysrescue)","uuid":"e054553a:fbccdcfe:0ae80bc8:9379377a","events":74,"device_table":[{"number":0,"major":254,"minor":1,"state":["active","sync"],"device":"/dev/vda1","raid_device":0},{"number":2,"major":254,"minor":2,"state":["writemostly","failfast","spare","rebuilding"],"device":"/dev/vda2","raid_device":1},{"number":3,"major":254,"minor":4,"state":["active","sync"],"device":"/dev/vda4","raid_device":1}],"array_size_num":101376,"used_dev_size_num":101376,"name_val":"sysrescue:126","uuid_val":"e054553a:fbccdcfe:0ae80bc8:9379377a","state_list":["clean","recovering"],"rebuild_status_percent":30,"creation_time_epoch":1659932149,"update_time_epoch":1659934062} diff --git a/tests/fixtures/generic/mdadm-query-raid1-syncing.json b/tests/fixtures/generic/mdadm-query-raid1-syncing.json new file mode 100644 index 000000000..7a6fb94e3 --- /dev/null +++ b/tests/fixtures/generic/mdadm-query-raid1-syncing.json @@ -0,0 +1 @@ +{"device":"/dev/md126","version":"1.2","creation_time":"Sun Aug 7 21:15:49 2022","raid_level":"raid1","array_size":"101376 (99.00 MiB 103.81 MB)","used_dev_size":"101376 (99.00 MiB 103.81 MB)","raid_devices":2,"total_devices":2,"persistence":"Superblock is persistent","update_time":"Sun Aug 7 21:15:49 2022","state":"clean, resyncing","active_devices":2,"working_devices":2,"failed_devices":0,"spare_devices":0,"consistency_policy":"resync","resync_status":"4% complete","name":"sysrescue:126 (local to host sysrescue)","uuid":"e054553a:fbccdcfe:0ae80bc8:9379377a","events":0,"device_table":[{"number":0,"major":254,"minor":1,"state":["active","sync"],"device":"/dev/vda1","raid_device":0},{"number":1,"major":254,"minor":2,"state":["active","sync"],"device":"/dev/vda2","raid_device":1}],"array_size_num":101376,"used_dev_size_num":101376,"name_val":"sysrescue:126","uuid_val":"e054553a:fbccdcfe:0ae80bc8:9379377a","state_list":["clean","resyncing"],"resync_status_percent":4,"creation_time_epoch":1659932149,"update_time_epoch":1659932149} diff --git a/tests/test_mdadm.py b/tests/test_mdadm.py new file mode 100644 index 000000000..dd7e68413 --- /dev/null +++ b/tests/test_mdadm.py @@ -0,0 +1,374 @@ +import os +import unittest +import json +import jc.parsers.mdadm + +THIS_DIR = os.path.dirname(os.path.abspath(__file__)) + + +class MyTests(unittest.TestCase): + + def setUp(self): + # input + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-examine-raid0-offline.out'), 'r', encoding='utf-8') as f: + self.mdadm_examine_raid0_offline = f.read() + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-examine-raid0-ok.out'), 'r', encoding='utf-8') as f: + self.mdadm_examine_raid0_ok = f.read() + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-examine-raid1-0-90-ok.out'), 'r', encoding='utf-8') as f: + self.mdadm_examine_raid1_0_90_ok = f.read() + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-examine-raid1-checking.out'), 'r', encoding='utf-8') as f: + self.mdadm_examine_raid1_checking = f.read() + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-examine-raid1-failfast.out'), 'r', encoding='utf-8') as f: + self.mdadm_examine_raid1_failfast = f.read() + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-examine-raid1-faulty1.out'), 'r', encoding='utf-8') as f: + self.mdadm_examine_raid1_faulty1 = f.read() + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-examine-raid1-faulty2.out'), 'r', encoding='utf-8') as f: + self.mdadm_examine_raid1_faulty2 = f.read() + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-examine-raid1-moreflags.out'), 'r', encoding='utf-8') as f: + self.mdadm_examine_raid1_moreflags = f.read() + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-examine-raid1-ok.out'), 'r', encoding='utf-8') as f: + self.mdadm_examine_raid1_ok = f.read() + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-examine-raid1-replacing.out'), 'r', encoding='utf-8') as f: + self.mdadm_examine_raid1_replacing = f.read() + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-examine-raid1-resync.out'), 'r', encoding='utf-8') as f: + self.mdadm_examine_raid1_resync = f.read() + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-examine-raid1-spare.out'), 'r', encoding='utf-8') as f: + self.mdadm_examine_raid1_spare = f.read() + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-examine-raid1-syncing.out'), 'r', encoding='utf-8') as f: + self.mdadm_examine_raid1_syncing = f.read() + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-examine.out'), 'r', encoding='utf-8') as f: + self.mdadm_examine = f.read() + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-query-detail.out'), 'r', encoding='utf-8') as f: + self.mdadm_query_detail = f.read() + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-query-raid0-ok.out'), 'r', encoding='utf-8') as f: + self.mdadm_query_raid0_ok = f.read() + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-query-raid1-failed-and-flags.out'), 'r', encoding='utf-8') as f: + self.mdadm_query_raid1_failed_and_flags = f.read() + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-query-raid1-faulty-and-removed.out'), 'r', encoding='utf-8') as f: + self.mdadm_query_raid1_faulty_and_removed = f.read() + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-query-raid1-faulty-with-spare.out'), 'r', encoding='utf-8') as f: + self.mdadm_query_raid1_faulty_with_spare = f.read() + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-query-raid1-faulty.out'), 'r', encoding='utf-8') as f: + self.mdadm_query_raid1_faulty = f.read() + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-query-raid1-ok-0-9.out'), 'r', encoding='utf-8') as f: + self.mdadm_query_raid1_ok_0_9 = f.read() + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-query-raid1-ok-failfast.out'), 'r', encoding='utf-8') as f: + self.mdadm_query_raid1_ok_failfast = f.read() + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-query-raid1-ok-spare.out'), 'r', encoding='utf-8') as f: + self.mdadm_query_raid1_ok_spare = f.read() + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-query-raid1-ok.out'), 'r', encoding='utf-8') as f: + self.mdadm_query_raid1_ok = f.read() + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-query-raid1-rebuild-failfast.out'), 'r', encoding='utf-8') as f: + self.mdadm_query_raid1_rebuild_failfast = f.read() + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-query-raid1-spare-writem-rebuild.out'), 'r', encoding='utf-8') as f: + self.mdadm_query_raid1_spare_writem_rebuild = f.read() + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-query-raid1-syncing.out'), 'r', encoding='utf-8') as f: + self.mdadm_query_raid1_syncing = f.read() + + # output + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-examine-raid0-offline.json'), 'r', encoding='utf-8') as f: + self.mdadm_examine_raid0_offline_json = json.loads(f.read()) + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-examine-raid0-ok.json'), 'r', encoding='utf-8') as f: + self.mdadm_examine_raid0_ok_json = json.loads(f.read()) + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-examine-raid1-0-90-ok.json'), 'r', encoding='utf-8') as f: + self.mdadm_examine_raid1_0_90_ok_json = json.loads(f.read()) + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-examine-raid1-checking.json'), 'r', encoding='utf-8') as f: + self.mdadm_examine_raid1_checking_json = json.loads(f.read()) + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-examine-raid1-failfast.json'), 'r', encoding='utf-8') as f: + self.mdadm_examine_raid1_failfast_json = json.loads(f.read()) + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-examine-raid1-faulty1.json'), 'r', encoding='utf-8') as f: + self.mdadm_examine_raid1_faulty1_json = json.loads(f.read()) + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-examine-raid1-faulty2.json'), 'r', encoding='utf-8') as f: + self.mdadm_examine_raid1_faulty2_json = json.loads(f.read()) + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-examine-raid1-moreflags.json'), 'r', encoding='utf-8') as f: + self.mdadm_examine_raid1_moreflags_json = json.loads(f.read()) + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-examine-raid1-ok.json'), 'r', encoding='utf-8') as f: + self.mdadm_examine_raid1_ok_json = json.loads(f.read()) + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-examine-raid1-replacing.json'), 'r', encoding='utf-8') as f: + self.mdadm_examine_raid1_replacing_json = json.loads(f.read()) + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-examine-raid1-resync.json'), 'r', encoding='utf-8') as f: + self.mdadm_examine_raid1_resync_json = json.loads(f.read()) + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-examine-raid1-spare.json'), 'r', encoding='utf-8') as f: + self.mdadm_examine_raid1_spare_json = json.loads(f.read()) + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-examine-raid1-syncing.json'), 'r', encoding='utf-8') as f: + self.mdadm_examine_raid1_syncing_json = json.loads(f.read()) + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-examine.json'), 'r', encoding='utf-8') as f: + self.mdadm_examine_json = json.loads(f.read()) + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-query-detail.json'), 'r', encoding='utf-8') as f: + self.mdadm_query_detial_json = json.loads(f.read()) + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-query-raid0-ok.json'), 'r', encoding='utf-8') as f: + self.mdadm_query_raid0_ok_json = json.loads(f.read()) + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-query-raid1-failed-and-flags.json'), 'r', encoding='utf-8') as f: + self.mdadm_query_raid1_failed_and_flags_json = json.loads(f.read()) + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-query-raid1-faulty-and-removed.json'), 'r', encoding='utf-8') as f: + self.mdadm_query_raid1_faulty_and_removed_json = json.loads(f.read()) + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-query-raid1-faulty-with-spare.json'), 'r', encoding='utf-8') as f: + self.mdadm_query_raid1_faulty_with_spare_json = json.loads(f.read()) + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-query-raid1-faulty.json'), 'r', encoding='utf-8') as f: + self.mdadm_query_raid1_faulty_json = json.loads(f.read()) + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-query-raid1-ok-0-9.json'), 'r', encoding='utf-8') as f: + self.mdadm_query_raid1_ok_0_9_json = json.loads(f.read()) + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-query-raid1-ok-failfast.json'), 'r', encoding='utf-8') as f: + self.mdadm_query_raid1_ok_failfast_json = json.loads(f.read()) + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-query-raid1-ok-spare.json'), 'r', encoding='utf-8') as f: + self.mdadm_query_raid1_ok_spare_json = json.loads(f.read()) + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-query-raid1-ok.json'), 'r', encoding='utf-8') as f: + self.mdadm_query_raid1_ok_json = json.loads(f.read()) + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-query-raid1-rebuild-failfast.json'), 'r', encoding='utf-8') as f: + self.mdadm_query_raid1_rebuild_failfast_json = json.loads(f.read()) + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-query-raid1-spare-writem-rebuild.json'), 'r', encoding='utf-8') as f: + self.mdadm_query_raid1_spare_writem_rebuild_json = json.loads(f.read()) + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-query-raid1-syncing.json'), 'r', encoding='utf-8') as f: + self.mdadm_query_raid1_syncing_json = json.loads(f.read()) + + + def test_mdadm_nodata(self): + """ + Test 'mdadm' with no data + """ + self.assertEqual(jc.parsers.mdadm.parse('', quiet=True), {}) + + + def test_mdadm_examine_raid0_offline(self): + """ + Test 'mdadm --examine' with offline RAID array + """ + self.assertEqual(jc.parsers.mdadm.parse(self.mdadm_examine_raid0_offline, quiet=True), self.mdadm_examine_raid0_offline_json) + + + def test_mdadm_examine_raid0_ok(self): + """ + Test 'mdadm --examine' with ok RAID array + """ + self.assertEqual(jc.parsers.mdadm.parse(self.mdadm_examine_raid0_ok, quiet=True), self.mdadm_examine_raid0_ok_json) + + + def test_mdadm_examine_raid1_0_90_ok(self): + """ + Test 'mdadm --examine' with ok RAID array v0.90 + """ + self.assertEqual(jc.parsers.mdadm.parse(self.mdadm_examine_raid1_0_90_ok, quiet=True), self.mdadm_examine_raid1_0_90_ok_json) + + + def test_mdadm_examine_raid1_checking(self): + """ + Test 'mdadm --examine' with checking RAID array + """ + self.assertEqual(jc.parsers.mdadm.parse(self.mdadm_examine_raid1_checking, quiet=True), self.mdadm_examine_raid1_checking_json) + + + def test_mdadm_examine_raid1_failfast(self): + """ + Test 'mdadm --examine' with failfast RAID array + """ + self.assertEqual(jc.parsers.mdadm.parse(self.mdadm_examine_raid1_failfast, quiet=True), self.mdadm_examine_raid1_failfast_json) + + + def test_mdadm_examine_raid1_faulty1(self): + """ + Test 'mdadm --examine' with faulty RAID array + """ + self.assertEqual(jc.parsers.mdadm.parse(self.mdadm_examine_raid1_faulty1, quiet=True), self.mdadm_examine_raid1_faulty1_json) + + + def test_mdadm_examine_raid1_faulty2(self): + """ + Test 'mdadm --examine' with faulty RAID array + """ + self.assertEqual(jc.parsers.mdadm.parse(self.mdadm_examine_raid1_faulty2, quiet=True), self.mdadm_examine_raid1_faulty2_json) + + + def test_mdadm_examine_raid1_moreflags(self): + """ + Test 'mdadm --examine' with RAID array with several flags + """ + self.assertEqual(jc.parsers.mdadm.parse(self.mdadm_examine_raid1_moreflags, quiet=True), self.mdadm_examine_raid1_moreflags_json) + + + def test_mdadm_examine_raid1_ok(self): + """ + Test 'mdadm --examine' with ok RAID array + """ + self.assertEqual(jc.parsers.mdadm.parse(self.mdadm_examine_raid1_ok, quiet=True), self.mdadm_examine_raid1_ok_json) + + + def test_mdadm_examine_raid1_replacing(self): + """ + Test 'mdadm --examine' with replacing RAID array + """ + self.assertEqual(jc.parsers.mdadm.parse(self.mdadm_examine_raid1_replacing, quiet=True), self.mdadm_examine_raid1_replacing_json) + + + def test_mdadm_examine_raid1_resync(self): + """ + Test 'mdadm --examine' with resyncing RAID array + """ + self.assertEqual(jc.parsers.mdadm.parse(self.mdadm_examine_raid1_resync, quiet=True), self.mdadm_examine_raid1_resync_json) + + + def test_mdadm_examine_raid1_spare(self): + """ + Test 'mdadm --examine' with spare in RAID array + """ + self.assertEqual(jc.parsers.mdadm.parse(self.mdadm_examine_raid1_spare, quiet=True), self.mdadm_examine_raid1_spare_json) + + + def test_mdadm_examine_raid1_syncing(self): + """ + Test 'mdadm --examine' with syncing RAID array + """ + self.assertEqual(jc.parsers.mdadm.parse(self.mdadm_examine_raid1_syncing, quiet=True), self.mdadm_examine_raid1_syncing_json) + + + def test_mdadm_examine(self): + """ + Test 'mdadm --examine' + """ + self.assertEqual(jc.parsers.mdadm.parse(self.mdadm_examine, quiet=True), self.mdadm_examine_json) + + + def test_mdadm_query_detail(self): + """ + Test 'mdadm --query --detail' + """ + self.assertEqual(jc.parsers.mdadm.parse(self.mdadm_query_detail, quiet=True), self.mdadm_query_detial_json) + + + def test_mdadm_query_raid0_ok(self): + """ + Test 'mdadm --query' on ok RAID array + """ + self.assertEqual(jc.parsers.mdadm.parse(self.mdadm_query_raid0_ok, quiet=True), self.mdadm_query_raid0_ok_json) + + + def test_mdadm_query_raid1_failed_and_flags(self): + """ + Test 'mdadm --query' on failed RAID array with flags + """ + self.assertEqual(jc.parsers.mdadm.parse(self.mdadm_query_raid1_failed_and_flags, quiet=True), self.mdadm_query_raid1_failed_and_flags_json) + + + def test_mdadm_query_raid1_faulty_and_removed(self): + """ + Test 'mdadm --query' on faulty RAID array with removed disk + """ + self.assertEqual(jc.parsers.mdadm.parse(self.mdadm_query_raid1_faulty_and_removed, quiet=True), self.mdadm_query_raid1_faulty_and_removed_json) + + + def test_mdadm_query_raid1_faulty_with_spare(self): + """ + Test 'mdadm --query' on faulty RAID array with spare disk + """ + self.assertEqual(jc.parsers.mdadm.parse(self.mdadm_query_raid1_faulty_with_spare, quiet=True), self.mdadm_query_raid1_faulty_with_spare_json) + + + def test_mdadm_query_raid1_faulty(self): + """ + Test 'mdadm --query' on faulty RAID + """ + self.assertEqual(jc.parsers.mdadm.parse(self.mdadm_query_raid1_faulty, quiet=True), self.mdadm_query_raid1_faulty_json) + + + def test_mdadm_query_raid1_ok_0_9(self): + """ + Test 'mdadm --query' on ok RAID on v0.9 + """ + self.assertEqual(jc.parsers.mdadm.parse(self.mdadm_query_raid1_ok_0_9, quiet=True), self.mdadm_query_raid1_ok_0_9_json) + + + def test_mdadm_query_raid1_ok_failfast(self): + """ + Test 'mdadm --query' on ok RAID with failfast + """ + self.assertEqual(jc.parsers.mdadm.parse(self.mdadm_query_raid1_ok_failfast, quiet=True), self.mdadm_query_raid1_ok_failfast_json) + + + def test_mdadm_query_raid1_ok_spare(self): + """ + Test 'mdadm --query' on ok RAID with spare + """ + self.assertEqual(jc.parsers.mdadm.parse(self.mdadm_query_raid1_ok_spare, quiet=True), self.mdadm_query_raid1_ok_spare_json) + + + def test_mdadm_query_raid1_ok(self): + """ + Test 'mdadm --query' on ok RAID + """ + self.assertEqual(jc.parsers.mdadm.parse(self.mdadm_query_raid1_ok, quiet=True), self.mdadm_query_raid1_ok_json) + + + def test_mdadm_query_raid1_rebuild_failfast(self): + """ + Test 'mdadm --query' on rebuilding RAID with failfast + """ + self.assertEqual(jc.parsers.mdadm.parse(self.mdadm_query_raid1_rebuild_failfast, quiet=True), self.mdadm_query_raid1_rebuild_failfast_json) + + + def test_mdadm_query_raid1_spare_writem_rebuild(self): + """ + Test 'mdadm --query' on rebuilding RAID with spare + """ + self.assertEqual(jc.parsers.mdadm.parse(self.mdadm_query_raid1_spare_writem_rebuild, quiet=True), self.mdadm_query_raid1_spare_writem_rebuild_json) + + + def test_mdadm_query_raid1_syncing(self): + """ + Test 'mdadm --query' on syncing RAID + """ + self.assertEqual(jc.parsers.mdadm.parse(self.mdadm_query_raid1_syncing, quiet=True), self.mdadm_query_raid1_syncing_json) + + +if __name__ == '__main__': + unittest.main() From 17ce8699398eff0f83616dcfd80127589f071261 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Mon, 15 Aug 2022 15:39:20 -0700 Subject: [PATCH 071/116] add warning for unparsable lines --- docs/parsers/syslog.md | 8 ++++++-- jc/parsers/syslog.py | 24 +++++++++++++++++++----- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/docs/parsers/syslog.md b/docs/parsers/syslog.md index 9ce7f353b..f5052d8b5 100644 --- a/docs/parsers/syslog.md +++ b/docs/parsers/syslog.md @@ -6,7 +6,8 @@ jc - JSON Convert Syslog RFC 5424 string parser This parser accepts a single syslog line string or multiple syslog lines -separated by newlines. +separated by newlines. A warning message to `STDERR` will be printed if an +unparsable line is found. The `timestamp_epoch` calculated timestamp field is naive. (i.e. based on the local time of the system the parser is run on) @@ -45,12 +46,15 @@ Blank values converted to `null`/`None` } } ], - "message": string + "message": string, + "unparsable": string # [2] } ] [0] naive timestamp if "timestamp" field is parsable, else null [1] timezone aware timestamp availabe for UTC, else null + [2] this field exists if the syslog line is not parsable. The value + is the original syslog line. Examples: diff --git a/jc/parsers/syslog.py b/jc/parsers/syslog.py index 938ce37d1..d2962804c 100644 --- a/jc/parsers/syslog.py +++ b/jc/parsers/syslog.py @@ -1,7 +1,8 @@ """jc - JSON Convert Syslog RFC 5424 string parser This parser accepts a single syslog line string or multiple syslog lines -separated by newlines. +separated by newlines. A warning message to `STDERR` will be printed if an +unparsable line is found. The `timestamp_epoch` calculated timestamp field is naive. (i.e. based on the local time of the system the parser is run on) @@ -40,12 +41,15 @@ } } ], - "message": string + "message": string, + "unparsable": string # [2] } ] [0] naive timestamp if "timestamp" field is parsable, else null [1] timezone aware timestamp availabe for UTC, else null + [2] this field exists if the syslog line is not parsable. The value + is the original syslog line. Examples: @@ -180,19 +184,19 @@ def _process(proc_data: List[Dict]) -> List[Dict]: item[key] = value.strip() # add timestamp fields - if item['timestamp']: + if 'timestamp' in item and item['timestamp']: format = (1300, 1310) dt = jc.utils.timestamp(item['timestamp'], format) item['timestamp_epoch'] = dt.naive item['timestamp_epoch_utc'] = dt.utc # fixup escaped characters - if item['message']: + if 'message' in item and item['message']: for esc, esc_sub in escape_map.items(): item['message'] = item['message'].replace(esc, esc_sub) # parse identity and key value pairs in the structured data section - if item['structured_data']: + if 'structured_data' in item and item['structured_data']: structs_list = [] structs = _extract_structs(item['structured_data']) @@ -292,6 +296,16 @@ def parse( 'message': syslog_dict['msg'] } + else: + syslog_out = { + 'unparsable': line + } + + if not quiet: + jc.utils.warning_message( + [f'Unparsable line found: {line}'] + ) + if syslog_out: raw_output.append(syslog_out) From b4b3a11f014c478a85320d1b510cb1289e8864f5 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Mon, 15 Aug 2022 16:54:02 -0700 Subject: [PATCH 072/116] tighten up key/value split --- jc/parsers/mdadm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jc/parsers/mdadm.py b/jc/parsers/mdadm.py index 52bb556bb..343974294 100644 --- a/jc/parsers/mdadm.py +++ b/jc/parsers/mdadm.py @@ -367,7 +367,7 @@ def parse( # key/value lines if ' : ' in line: - key, value = line.split(' : ') + key, value = line.split(' : ', maxsplit=1) key = key.strip().lower().replace(' ', '_') value = value.strip() raw_output[key] = value From 7c4cf66243c4896cf406ad9f94743a09c042af08 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Mon, 15 Aug 2022 16:54:10 -0700 Subject: [PATCH 073/116] add syslog tests --- tests/fixtures/generic/syslog-5424.json | 1 + tests/fixtures/generic/syslog-5424.out | 12 +++++++++ tests/test_syslog.py | 35 +++++++++++++++++++++++++ 3 files changed, 48 insertions(+) create mode 100644 tests/fixtures/generic/syslog-5424.json create mode 100644 tests/fixtures/generic/syslog-5424.out create mode 100644 tests/test_syslog.py diff --git a/tests/fixtures/generic/syslog-5424.json b/tests/fixtures/generic/syslog-5424.json new file mode 100644 index 000000000..42528945c --- /dev/null +++ b/tests/fixtures/generic/syslog-5424.json @@ -0,0 +1 @@ +[{"priority":165,"version":1,"timestamp":"2003-08-24T05:14:15.000003-07:00","hostname":"192.0.2.1","appname":"myproc","proc_id":8710,"msg_id":null,"structured_data":null,"message":"%% It's time to make the do-nuts.","timestamp_epoch":1061727255,"timestamp_epoch_utc":null},{"unparsable":"<34>1 203-10-11T22:14:15.003Z mymachine.example.com su - ID47 - BOM'su root' failed for lonvick on /dev/pts/8"},{"priority":34,"version":1,"timestamp":"2003-10-11T22:14:15.003Z","hostname":"mymachine.example.com","appname":"su","proc_id":null,"msg_id":"ID47","structured_data":null,"message":"BOM'su root' failed for lonvick on /dev/pts/8","timestamp_epoch":1065935655,"timestamp_epoch_utc":1065910455},{"priority":165,"version":1,"timestamp":"2003-08-24T05:14:15.000003-07:00","hostname":"192.0.2.1","appname":"myproc","proc_id":8710,"msg_id":null,"structured_data":null,"message":"%% It's time to make the do-nuts.","timestamp_epoch":1061727255,"timestamp_epoch_utc":null},{"priority":null,"version":1,"timestamp":"2003-10-11T22:14:15.003Z","hostname":"mymachine.example.com","appname":"su","proc_id":null,"msg_id":"ID47","structured_data":null,"message":"BOM'su root' failed for lonvick on /dev/pts/8","timestamp_epoch":1065935655,"timestamp_epoch_utc":1065910455},{"priority":190,"version":1,"timestamp":"2003-10-11T22:14:15.003Z","hostname":"mymachine.example.com","appname":"evntslog","proc_id":null,"msg_id":"ID47","structured_data":[{"identity":"exampleSDID@32473","parameters":{"iut":"3","eventSource":"Application","eventID":"10] 11"}}],"message":"BOMAn application event log entry..[ ] sadasd","timestamp_epoch":1065935655,"timestamp_epoch_utc":1065910455},{"unparsable":"<190)1 2003-10-11T22:14:15.003Z mymachine.example.com evntslog - ID47 [exampleSDID@32473 iut=\"3\" eventSource=\"Application\" eventID=\"10\\] 11\"] BOMAn application event log entry..[ ] sadasd"},{"priority":null,"version":null,"timestamp":"2003-10-11T22:14:15.003Z","hostname":"mymachine.example.com","appname":"evntslog","proc_id":null,"msg_id":"ID47","structured_data":[{"identity":"exampleSDID@32473","parameters":{"iut":"3","eventSource":"Application","eventID":"1011"}},{"identity":"examplePriority@32473","parameters":{"class":"high"}}],"message":null,"timestamp_epoch":1065935655,"timestamp_epoch_utc":1065910455},{"priority":1,"version":12,"timestamp":null,"hostname":"mymachine","appname":null,"proc_id":null,"msg_id":"ID47","structured_data":null,"message":"asd asdaasd"}] diff --git a/tests/fixtures/generic/syslog-5424.out b/tests/fixtures/generic/syslog-5424.out new file mode 100644 index 000000000..93f7174b5 --- /dev/null +++ b/tests/fixtures/generic/syslog-5424.out @@ -0,0 +1,12 @@ +<165>1 2003-08-24T05:14:15.000003-07:00 192.0.2.1 myproc 8710 - - %% It's time to make the do-nuts. +<34>1 203-10-11T22:14:15.003Z mymachine.example.com su - ID47 - BOM'su root' failed for lonvick on /dev/pts/8 +<34>1 2003-10-11T22:14:15.003Z mymachine.example.com su - ID47 - BOM'su root' failed for lonvick on /dev/pts/8 + +<165>1 2003-08-24T05:14:15.000003-07:00 192.0.2.1 myproc 8710 - - %% It's time to make the do-nuts. +1 2003-10-11T22:14:15.003Z mymachine.example.com su - ID47 - BOM'su root' failed for lonvick on /dev/pts/8 +<190>1 2003-10-11T22:14:15.003Z mymachine.example.com evntslog - ID47 [exampleSDID@32473 iut="3" eventSource="Application" eventID="10\] 11"] BOMAn application event log entry..[ ] sadasd + +<190)1 2003-10-11T22:14:15.003Z mymachine.example.com evntslog - ID47 [exampleSDID@32473 iut="3" eventSource="Application" eventID="10\] 11"] BOMAn application event log entry..[ ] sadasd +2003-10-11T22:14:15.003Z mymachine.example.com evntslog - ID47 [exampleSDID@32473 iut="3" eventSource="Application" eventID="1011"][examplePriority@32473 class="high"] +<1>12 - mymachine - - ID47 - asd asdaasd + diff --git a/tests/test_syslog.py b/tests/test_syslog.py new file mode 100644 index 000000000..2a05e4a40 --- /dev/null +++ b/tests/test_syslog.py @@ -0,0 +1,35 @@ +import os +import unittest +import json +import jc.parsers.syslog + +THIS_DIR = os.path.dirname(os.path.abspath(__file__)) + + +class MyTests(unittest.TestCase): + + def setUp(self): + # input + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/syslog-5424.out'), 'r', encoding='utf-8') as f: + self.syslog = f.read() + + # output + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/syslog-5424.json'), 'r', encoding='utf-8') as f: + self.syslog_json = json.loads(f.read()) + + + def test_syslog_nodata(self): + """ + Test 'syslog' with no data + """ + self.assertEqual(jc.parsers.syslog.parse('', quiet=True), []) + + def test_syslog_sample(self): + """ + Test 'syslog' with sample data + """ + self.assertEqual(jc.parsers.syslog.parse(self.syslog, quiet=True), self.syslog_json) + + +if __name__ == '__main__': + unittest.main() From e7c8778e3087fb3f7bbe2fde89800dc1c48d0ed8 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Mon, 15 Aug 2022 18:10:43 -0700 Subject: [PATCH 074/116] add docs, int conversion, and better rfc compliance --- docs/parsers/syslog_bsd.md | 46 +++++++++++++++------- jc/parsers/syslog_bsd.py | 79 +++++++++++++++++++++++++++----------- 2 files changed, 90 insertions(+), 35 deletions(-) diff --git a/docs/parsers/syslog_bsd.md b/docs/parsers/syslog_bsd.md index a3f9ecb02..824d62c63 100644 --- a/docs/parsers/syslog_bsd.md +++ b/docs/parsers/syslog_bsd.md @@ -5,15 +5,13 @@ jc - JSON Convert Syslog RFC 3164 string parser -<> +This parser accepts a single syslog line string or multiple syslog lines +separated by newlines. A warning message to `STDERR` will be printed if an +unparsable line is found. Usage (cli): - $ syslogstring | jc --syslog-bsd - -or - - $ jc syslog-3164 + $ echo '<34>Oct 11 22:14:15 mymachine su: su root...' | jc --syslog-bsd Usage (module): @@ -24,19 +22,41 @@ Schema: [ { - "syslog-3164": string, - "bar": boolean, - "baz": integer + "priority": integer/null, + "date": string, + "hostname": string, + "tag": string, + "content": string, + "unparsable": string, # [0] } ] + [0] this field exists if the syslog line is not parsable. The value + is the original syslog line. + Examples: - $ syslog-3164 | jc --syslog-3164 -p - [] + $ cat syslog.txt | jc --syslog-bsd -p + [ + { + "priority": 34, + "date": "Oct 11 22:14:15", + "hostname": "mymachine", + "tag": "su", + "content": "'su root' failed for lonvick on /dev/pts/8" + } + ] - $ syslog-3164 | jc --syslog-3164 -p -r - [] + $ cat syslog.txt | jc --syslog-bsd -p -r + [ + { + "priority": "34", + "date": "Oct 11 22:14:15", + "hostname": "mymachine", + "tag": "su", + "content": "'su root' failed for lonvick on /dev/pts/8" + } + ] diff --git a/jc/parsers/syslog_bsd.py b/jc/parsers/syslog_bsd.py index 32ad55a10..2a4406b95 100644 --- a/jc/parsers/syslog_bsd.py +++ b/jc/parsers/syslog_bsd.py @@ -1,14 +1,12 @@ """jc - JSON Convert Syslog RFC 3164 string parser -<> +This parser accepts a single syslog line string or multiple syslog lines +separated by newlines. A warning message to `STDERR` will be printed if an +unparsable line is found. Usage (cli): - $ syslogstring | jc --syslog-bsd - -or - - $ jc syslog-3164 + $ echo '<34>Oct 11 22:14:15 mymachine su: su root...' | jc --syslog-bsd Usage (module): @@ -19,19 +17,41 @@ [ { - "syslog-3164": string, - "bar": boolean, - "baz": integer + "priority": integer/null, + "date": string, + "hostname": string, + "tag": string, + "content": string, + "unparsable": string, # [0] } ] + [0] this field exists if the syslog line is not parsable. The value + is the original syslog line. + Examples: - $ syslog-3164 | jc --syslog-3164 -p - [] + $ cat syslog.txt | jc --syslog-bsd -p + [ + { + "priority": 34, + "date": "Oct 11 22:14:15", + "hostname": "mymachine", + "tag": "su", + "content": "'su root' failed for lonvick on /dev/pts/8" + } + ] - $ syslog-3164 | jc --syslog-3164 -p -r - [] + $ cat syslog.txt | jc --syslog-bsd -p -r + [ + { + "priority": "34", + "date": "Oct 11 22:14:15", + "hostname": "mymachine", + "tag": "su", + "content": "'su root' failed for lonvick on /dev/pts/8" + } + ] """ import re from typing import List, Dict @@ -61,11 +81,12 @@ def _process(proc_data: List[Dict]) -> List[Dict]: List of Dictionaries. Structured to conform to the schema. """ + int_list = {'priority'} - # process the data here - # rebuild output for added semantic information - # use helper functions in jc.utils for int, float, bool - # conversions and timestamps + for item in proc_data: + for key in item: + if key in int_list: + item[key] = jc.utils.convert_to_int(item[key]) return proc_data @@ -92,14 +113,17 @@ def parse( jc.utils.input_type_check(data) raw_output: List = [] + syslog_dict = {} # inspired by https://gist.github.com/miticojo/b16bb13e78572c2d2fac82d9516d5c32 syslog = re.compile(r''' (?P<\d*>)? - (?P[A-Z][a-z][a-z]\s{1,2}\d{1,2}\s\d{2}?:\d{2}:\d{2})\s - (?P[\w][\w\d\.@-]*)\s - (?P[\w\d\[\]\.@-]+):?\s - (?P.*) + (?P[A-Z][a-z][a-z]\s{1,2}\d{1,2}\s\d{2}?:\d{2}:\d{2})?\s + (?P[\w][\w\d\.:@-]*)?\s + (?P + (?P\w+)? + (?P.*) + ) ''', re.VERBOSE ) @@ -116,10 +140,21 @@ def parse( 'priority': priority, 'date': syslog_match.group('date'), 'hostname': syslog_match.group('host'), + # 'raw_msg': syslog_match.group('msg'), 'tag': syslog_match.group('tag'), - 'message': syslog_match.group('message') + 'content': syslog_match.group('content').lstrip(' :').rstrip() } + else: + syslog_dict = { + 'unparsable': line + } + + if not quiet: + jc.utils.warning_message( + [f'Unparsable line found: {line}'] + ) + if syslog_dict: raw_output.append(syslog_dict) From c82c6a88f8c3c388796d36d22f6eb3ce1a32c281 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Mon, 15 Aug 2022 18:10:48 -0700 Subject: [PATCH 075/116] add tests --- tests/fixtures/generic/syslog-3164.json | 1 + tests/fixtures/generic/syslog-3164.out | 34 ++++++++++++++++++++++++ tests/test_syslog_bsd.py | 35 +++++++++++++++++++++++++ 3 files changed, 70 insertions(+) create mode 100644 tests/fixtures/generic/syslog-3164.json create mode 100644 tests/fixtures/generic/syslog-3164.out create mode 100644 tests/test_syslog_bsd.py diff --git a/tests/fixtures/generic/syslog-3164.json b/tests/fixtures/generic/syslog-3164.json new file mode 100644 index 000000000..378843c7c --- /dev/null +++ b/tests/fixtures/generic/syslog-3164.json @@ -0,0 +1 @@ +[{"priority":34,"date":"Oct 11 22:14:15","hostname":"mymachine","tag":"su","content":"'su root' failed for lonvick on /dev/pts/8"},{"priority":null,"date":"Oct 11 22:14:15","hostname":"mymachine","tag":"su","content":"'su root' failed for lonvick on /dev/pts/8"},{"priority":35,"date":"Oct 12 22:14:15","hostname":"client_machine","tag":"su","content":"'su root' failed for joe on /dev/pts/2"},{"priority":35,"date":"Mar 7 04:02:16","hostname":"avas","tag":"clamd","content":"[11165]: /var/amavis/amavis-20040307T033734-10329/parts/part-00003: Worm.Mydoom.F FOUND"},{"priority":null,"date":"Mar 7 04:05:55","hostname":"avas","tag":"clamd","content":"[11240]: /var/amavis/amavis-20040307T035901-10615/parts/part-00002: Worm.SomeFool.Gen-1 FOUND"},{"priority":5,"date":"Mar 7 09:00:51","hostname":"avas","tag":"clamd","content":"[27173]: SelfCheck: Database status OK."},{"priority":null,"date":"Mar 7 05:59:02","hostname":"avas","tag":"clamd","content":"[27173]: Database correctly reloaded (20400 viruses)"},{"priority":null,"date":"Mar 7 04:02:16","hostname":"avas","tag":"clamd","content":"[11165]: /var/amavis/amavis-20040307T033734-10329/parts/part-00003: Worm.Mydoom.F FOUND"},{"priority":null,"date":"Mar 7 04:05:55","hostname":"avas","tag":"clamd","content":"[11240]: /var/amavis/amavis-20040307T035901-10615/parts/part-00002: Worm.SomeFool.Gen-1 FOUND"},{"priority":null,"date":"Mar 7 09:00:51","hostname":"avas","tag":"clamd","content":"[27173]: SelfCheck: Database status OK."},{"priority":null,"date":"Mar 7 05:59:02","hostname":"avas","tag":"clamd","content":"[27173]: Database correctly reloaded (20400 viruses)"},{"priority":null,"date":"Mar 7 11:14:35","hostname":"avas","tag":"dccd","content":"[13284]: 21 requests/sec are too many from anonymous 205.201.1.56,2246"},{"priority":null,"date":"Mar 8 00:22:57","hostname":"avas","tag":"dccifd","content":"[9933]: write(MTA socket,4): Broken pipe"},{"priority":null,"date":"Mar 7 21:23:22","hostname":"avas","tag":"dccifd","content":"[6191]: missing message body"},{"priority":null,"date":"Mar 9 16:05:17","hostname":"avas","tag":"named","content":"[12045]: zone PLNet/IN: refresh: non-authoritative answer from master 10.0.0.253#53"},{"priority":null,"date":"Mar 10 00:38:16","hostname":"avas","tag":"dccifd","content":"[23069]: continue not asking DCC 17 seconds after failure"},{"priority":null,"date":"Mar 10 09:42:11","hostname":"avas","tag":"named","content":"client 127.0.0.1#55524: query: 23.68.27.142.sa-trusted.bondedsender.org IN TXT"},{"priority":null,"date":"Mar 9 03:48:07","hostname":"avas","tag":"dccd","content":"[145]: automatic dbclean; starting `dbclean -DPq -i 1189 -L info,local5.notice -L error,local5.err`"},{"priority":null,"date":"Mar 9 11:58:18","hostname":"avas","tag":"kernel","content":"i810_audio: Connection 0 with codec id 2"},{"priority":null,"date":"Mar 9 19:41:13","hostname":"avas","tag":"dccd","content":"[3004]: \"packet length 44 too small for REPORT\" sent to client 1 at 194.63.250.215,47577"},{"priority":null,"date":"Mar 8 09:01:07","hostname":"avas","tag":"sshd","content":"(pam_unix)[21839]: session opened for user tom by (uid=35567)"},{"priority":null,"date":"Mar 8 03:52:04","hostname":"avas","tag":"dccd","content":"[13284]: 1.2.32 database /home/dcc/dcc_db reopened with 997 MByte window"},{"priority":null,"date":"Mar 8 16:05:26","hostname":"avas","tag":"arpwatch","content":"listening on eth0"},{"priority":null,"date":"Mar 10 10:00:06","hostname":"avas","tag":"named","content":"[6986]: zone PLNet/IN: refresh: non-authoritative answer from master 192.75.26.21#53"},{"priority":null,"date":"Mar 10 10:00:10","hostname":"avas","tag":"named","content":"[6986]: client 127.0.0.1#55867: query: mail.canfor.ca IN MX"},{"priority":null,"date":"Mar 8 15:18:40","hostname":"avas:","tag":"last","content":"message repeated 11 times"},{"priority":null,"date":"Mar 8 15:18:40","hostname":"127:0:ab::1","tag":"sshd","content":"unauthorized request"},{"priority":null,"date":"Mar 8 15:18:40","hostname":"server.example.com","tag":"sshd","content":"unauthorized request"},{"priority":null,"date":"Mar 8 15:18:40","hostname":"192.168.1.1","tag":"sshd","content":"unauthorized request"},{"priority":35,"date":"Mar 8 15:18:40","hostname":"server.example.com","tag":"sshd","content":"unauthorized request"},{"unparsable":"<7>unparsable line"}] diff --git a/tests/fixtures/generic/syslog-3164.out b/tests/fixtures/generic/syslog-3164.out new file mode 100644 index 000000000..d14de4870 --- /dev/null +++ b/tests/fixtures/generic/syslog-3164.out @@ -0,0 +1,34 @@ +<34>Oct 11 22:14:15 mymachine su: 'su root' failed for lonvick on /dev/pts/8 +Oct 11 22:14:15 mymachine su: 'su root' failed for lonvick on /dev/pts/8 +<35>Oct 12 22:14:15 client_machine su: 'su root' failed for joe on /dev/pts/2 +<35>Mar 7 04:02:16 avas clamd[11165]: /var/amavis/amavis-20040307T033734-10329/parts/part-00003: Worm.Mydoom.F FOUND +Mar 7 04:05:55 avas clamd[11240]: /var/amavis/amavis-20040307T035901-10615/parts/part-00002: Worm.SomeFool.Gen-1 FOUND +<5>Mar 7 09:00:51 avas clamd[27173]: SelfCheck: Database status OK. +Mar 7 05:59:02 avas clamd[27173]: Database correctly reloaded (20400 viruses) +Mar 7 04:02:16 avas clamd[11165]: /var/amavis/amavis-20040307T033734-10329/parts/part-00003: Worm.Mydoom.F FOUND +Mar 7 04:05:55 avas clamd[11240]: /var/amavis/amavis-20040307T035901-10615/parts/part-00002: Worm.SomeFool.Gen-1 FOUND +Mar 7 09:00:51 avas clamd[27173]: SelfCheck: Database status OK. +Mar 7 05:59:02 avas clamd[27173]: Database correctly reloaded (20400 viruses) +Mar 7 11:14:35 avas dccd[13284]: 21 requests/sec are too many from anonymous 205.201.1.56,2246 +Mar 8 00:22:57 avas dccifd[9933]: write(MTA socket,4): Broken pipe +Mar 7 21:23:22 avas dccifd[6191]: missing message body +Mar 9 16:05:17 avas named[12045]: zone PLNet/IN: refresh: non-authoritative answer from master 10.0.0.253#53 +Mar 10 00:38:16 avas dccifd[23069]: continue not asking DCC 17 seconds after failure + + +Mar 10 09:42:11 avas named: client 127.0.0.1#55524: query: 23.68.27.142.sa-trusted.bondedsender.org IN TXT +Mar 9 03:48:07 avas dccd[145]: automatic dbclean; starting `dbclean -DPq -i 1189 -L info,local5.notice -L error,local5.err` +Mar 9 11:58:18 avas kernel: i810_audio: Connection 0 with codec id 2 +Mar 9 19:41:13 avas dccd[3004]: "packet length 44 too small for REPORT" sent to client 1 at 194.63.250.215,47577 +Mar 8 09:01:07 avas sshd(pam_unix)[21839]: session opened for user tom by (uid=35567) + +Mar 8 03:52:04 avas dccd[13284]: 1.2.32 database /home/dcc/dcc_db reopened with 997 MByte window +Mar 8 16:05:26 avas arpwatch: listening on eth0 +Mar 10 10:00:06 avas named[6986]: zone PLNet/IN: refresh: non-authoritative answer from master 192.75.26.21#53 +Mar 10 10:00:10 avas named[6986]: client 127.0.0.1#55867: query: mail.canfor.ca IN MX +Mar 8 15:18:40 avas: last message repeated 11 times +Mar 8 15:18:40 127:0:ab::1 sshd: unauthorized request +Mar 8 15:18:40 server.example.com sshd: unauthorized request +Mar 8 15:18:40 192.168.1.1 sshd: unauthorized request +<35>Mar 8 15:18:40 server.example.com sshd: unauthorized request +<7>unparsable line diff --git a/tests/test_syslog_bsd.py b/tests/test_syslog_bsd.py new file mode 100644 index 000000000..ff0516c96 --- /dev/null +++ b/tests/test_syslog_bsd.py @@ -0,0 +1,35 @@ +import os +import unittest +import json +import jc.parsers.syslog_bsd + +THIS_DIR = os.path.dirname(os.path.abspath(__file__)) + + +class MyTests(unittest.TestCase): + + def setUp(self): + # input + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/syslog-3164.out'), 'r', encoding='utf-8') as f: + self.syslog = f.read() + + # output + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/syslog-3164.json'), 'r', encoding='utf-8') as f: + self.syslog_json = json.loads(f.read()) + + + def test_syslog_bsd_nodata(self): + """ + Test 'syslog_bsd' with no data + """ + self.assertEqual(jc.parsers.syslog_bsd.parse('', quiet=True), []) + + def test_syslog_bsd_sample(self): + """ + Test 'syslog_bsd' with sample data + """ + self.assertEqual(jc.parsers.syslog_bsd.parse(self.syslog, quiet=True), self.syslog_json) + + +if __name__ == '__main__': + unittest.main() From d42ec58ad163b296a003c797221c6d93df38d5ac Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Mon, 15 Aug 2022 18:21:58 -0700 Subject: [PATCH 076/116] remove colon at the end of the hostname if it exists. --- jc/parsers/syslog_bsd.py | 2 +- tests/fixtures/generic/syslog-3164.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/jc/parsers/syslog_bsd.py b/jc/parsers/syslog_bsd.py index 2a4406b95..e6fe3f127 100644 --- a/jc/parsers/syslog_bsd.py +++ b/jc/parsers/syslog_bsd.py @@ -139,7 +139,7 @@ def parse( syslog_dict = { 'priority': priority, 'date': syslog_match.group('date'), - 'hostname': syslog_match.group('host'), + 'hostname': syslog_match.group('host').rstrip(':'), # 'raw_msg': syslog_match.group('msg'), 'tag': syslog_match.group('tag'), 'content': syslog_match.group('content').lstrip(' :').rstrip() diff --git a/tests/fixtures/generic/syslog-3164.json b/tests/fixtures/generic/syslog-3164.json index 378843c7c..667c4743f 100644 --- a/tests/fixtures/generic/syslog-3164.json +++ b/tests/fixtures/generic/syslog-3164.json @@ -1 +1 @@ -[{"priority":34,"date":"Oct 11 22:14:15","hostname":"mymachine","tag":"su","content":"'su root' failed for lonvick on /dev/pts/8"},{"priority":null,"date":"Oct 11 22:14:15","hostname":"mymachine","tag":"su","content":"'su root' failed for lonvick on /dev/pts/8"},{"priority":35,"date":"Oct 12 22:14:15","hostname":"client_machine","tag":"su","content":"'su root' failed for joe on /dev/pts/2"},{"priority":35,"date":"Mar 7 04:02:16","hostname":"avas","tag":"clamd","content":"[11165]: /var/amavis/amavis-20040307T033734-10329/parts/part-00003: Worm.Mydoom.F FOUND"},{"priority":null,"date":"Mar 7 04:05:55","hostname":"avas","tag":"clamd","content":"[11240]: /var/amavis/amavis-20040307T035901-10615/parts/part-00002: Worm.SomeFool.Gen-1 FOUND"},{"priority":5,"date":"Mar 7 09:00:51","hostname":"avas","tag":"clamd","content":"[27173]: SelfCheck: Database status OK."},{"priority":null,"date":"Mar 7 05:59:02","hostname":"avas","tag":"clamd","content":"[27173]: Database correctly reloaded (20400 viruses)"},{"priority":null,"date":"Mar 7 04:02:16","hostname":"avas","tag":"clamd","content":"[11165]: /var/amavis/amavis-20040307T033734-10329/parts/part-00003: Worm.Mydoom.F FOUND"},{"priority":null,"date":"Mar 7 04:05:55","hostname":"avas","tag":"clamd","content":"[11240]: /var/amavis/amavis-20040307T035901-10615/parts/part-00002: Worm.SomeFool.Gen-1 FOUND"},{"priority":null,"date":"Mar 7 09:00:51","hostname":"avas","tag":"clamd","content":"[27173]: SelfCheck: Database status OK."},{"priority":null,"date":"Mar 7 05:59:02","hostname":"avas","tag":"clamd","content":"[27173]: Database correctly reloaded (20400 viruses)"},{"priority":null,"date":"Mar 7 11:14:35","hostname":"avas","tag":"dccd","content":"[13284]: 21 requests/sec are too many from anonymous 205.201.1.56,2246"},{"priority":null,"date":"Mar 8 00:22:57","hostname":"avas","tag":"dccifd","content":"[9933]: write(MTA socket,4): Broken pipe"},{"priority":null,"date":"Mar 7 21:23:22","hostname":"avas","tag":"dccifd","content":"[6191]: missing message body"},{"priority":null,"date":"Mar 9 16:05:17","hostname":"avas","tag":"named","content":"[12045]: zone PLNet/IN: refresh: non-authoritative answer from master 10.0.0.253#53"},{"priority":null,"date":"Mar 10 00:38:16","hostname":"avas","tag":"dccifd","content":"[23069]: continue not asking DCC 17 seconds after failure"},{"priority":null,"date":"Mar 10 09:42:11","hostname":"avas","tag":"named","content":"client 127.0.0.1#55524: query: 23.68.27.142.sa-trusted.bondedsender.org IN TXT"},{"priority":null,"date":"Mar 9 03:48:07","hostname":"avas","tag":"dccd","content":"[145]: automatic dbclean; starting `dbclean -DPq -i 1189 -L info,local5.notice -L error,local5.err`"},{"priority":null,"date":"Mar 9 11:58:18","hostname":"avas","tag":"kernel","content":"i810_audio: Connection 0 with codec id 2"},{"priority":null,"date":"Mar 9 19:41:13","hostname":"avas","tag":"dccd","content":"[3004]: \"packet length 44 too small for REPORT\" sent to client 1 at 194.63.250.215,47577"},{"priority":null,"date":"Mar 8 09:01:07","hostname":"avas","tag":"sshd","content":"(pam_unix)[21839]: session opened for user tom by (uid=35567)"},{"priority":null,"date":"Mar 8 03:52:04","hostname":"avas","tag":"dccd","content":"[13284]: 1.2.32 database /home/dcc/dcc_db reopened with 997 MByte window"},{"priority":null,"date":"Mar 8 16:05:26","hostname":"avas","tag":"arpwatch","content":"listening on eth0"},{"priority":null,"date":"Mar 10 10:00:06","hostname":"avas","tag":"named","content":"[6986]: zone PLNet/IN: refresh: non-authoritative answer from master 192.75.26.21#53"},{"priority":null,"date":"Mar 10 10:00:10","hostname":"avas","tag":"named","content":"[6986]: client 127.0.0.1#55867: query: mail.canfor.ca IN MX"},{"priority":null,"date":"Mar 8 15:18:40","hostname":"avas:","tag":"last","content":"message repeated 11 times"},{"priority":null,"date":"Mar 8 15:18:40","hostname":"127:0:ab::1","tag":"sshd","content":"unauthorized request"},{"priority":null,"date":"Mar 8 15:18:40","hostname":"server.example.com","tag":"sshd","content":"unauthorized request"},{"priority":null,"date":"Mar 8 15:18:40","hostname":"192.168.1.1","tag":"sshd","content":"unauthorized request"},{"priority":35,"date":"Mar 8 15:18:40","hostname":"server.example.com","tag":"sshd","content":"unauthorized request"},{"unparsable":"<7>unparsable line"}] +[{"priority":34,"date":"Oct 11 22:14:15","hostname":"mymachine","tag":"su","content":"'su root' failed for lonvick on /dev/pts/8"},{"priority":null,"date":"Oct 11 22:14:15","hostname":"mymachine","tag":"su","content":"'su root' failed for lonvick on /dev/pts/8"},{"priority":35,"date":"Oct 12 22:14:15","hostname":"client_machine","tag":"su","content":"'su root' failed for joe on /dev/pts/2"},{"priority":35,"date":"Mar 7 04:02:16","hostname":"avas","tag":"clamd","content":"[11165]: /var/amavis/amavis-20040307T033734-10329/parts/part-00003: Worm.Mydoom.F FOUND"},{"priority":null,"date":"Mar 7 04:05:55","hostname":"avas","tag":"clamd","content":"[11240]: /var/amavis/amavis-20040307T035901-10615/parts/part-00002: Worm.SomeFool.Gen-1 FOUND"},{"priority":5,"date":"Mar 7 09:00:51","hostname":"avas","tag":"clamd","content":"[27173]: SelfCheck: Database status OK."},{"priority":null,"date":"Mar 7 05:59:02","hostname":"avas","tag":"clamd","content":"[27173]: Database correctly reloaded (20400 viruses)"},{"priority":null,"date":"Mar 7 04:02:16","hostname":"avas","tag":"clamd","content":"[11165]: /var/amavis/amavis-20040307T033734-10329/parts/part-00003: Worm.Mydoom.F FOUND"},{"priority":null,"date":"Mar 7 04:05:55","hostname":"avas","tag":"clamd","content":"[11240]: /var/amavis/amavis-20040307T035901-10615/parts/part-00002: Worm.SomeFool.Gen-1 FOUND"},{"priority":null,"date":"Mar 7 09:00:51","hostname":"avas","tag":"clamd","content":"[27173]: SelfCheck: Database status OK."},{"priority":null,"date":"Mar 7 05:59:02","hostname":"avas","tag":"clamd","content":"[27173]: Database correctly reloaded (20400 viruses)"},{"priority":null,"date":"Mar 7 11:14:35","hostname":"avas","tag":"dccd","content":"[13284]: 21 requests/sec are too many from anonymous 205.201.1.56,2246"},{"priority":null,"date":"Mar 8 00:22:57","hostname":"avas","tag":"dccifd","content":"[9933]: write(MTA socket,4): Broken pipe"},{"priority":null,"date":"Mar 7 21:23:22","hostname":"avas","tag":"dccifd","content":"[6191]: missing message body"},{"priority":null,"date":"Mar 9 16:05:17","hostname":"avas","tag":"named","content":"[12045]: zone PLNet/IN: refresh: non-authoritative answer from master 10.0.0.253#53"},{"priority":null,"date":"Mar 10 00:38:16","hostname":"avas","tag":"dccifd","content":"[23069]: continue not asking DCC 17 seconds after failure"},{"priority":null,"date":"Mar 10 09:42:11","hostname":"avas","tag":"named","content":"client 127.0.0.1#55524: query: 23.68.27.142.sa-trusted.bondedsender.org IN TXT"},{"priority":null,"date":"Mar 9 03:48:07","hostname":"avas","tag":"dccd","content":"[145]: automatic dbclean; starting `dbclean -DPq -i 1189 -L info,local5.notice -L error,local5.err`"},{"priority":null,"date":"Mar 9 11:58:18","hostname":"avas","tag":"kernel","content":"i810_audio: Connection 0 with codec id 2"},{"priority":null,"date":"Mar 9 19:41:13","hostname":"avas","tag":"dccd","content":"[3004]: \"packet length 44 too small for REPORT\" sent to client 1 at 194.63.250.215,47577"},{"priority":null,"date":"Mar 8 09:01:07","hostname":"avas","tag":"sshd","content":"(pam_unix)[21839]: session opened for user tom by (uid=35567)"},{"priority":null,"date":"Mar 8 03:52:04","hostname":"avas","tag":"dccd","content":"[13284]: 1.2.32 database /home/dcc/dcc_db reopened with 997 MByte window"},{"priority":null,"date":"Mar 8 16:05:26","hostname":"avas","tag":"arpwatch","content":"listening on eth0"},{"priority":null,"date":"Mar 10 10:00:06","hostname":"avas","tag":"named","content":"[6986]: zone PLNet/IN: refresh: non-authoritative answer from master 192.75.26.21#53"},{"priority":null,"date":"Mar 10 10:00:10","hostname":"avas","tag":"named","content":"[6986]: client 127.0.0.1#55867: query: mail.canfor.ca IN MX"},{"priority":null,"date":"Mar 8 15:18:40","hostname":"avas","tag":"last","content":"message repeated 11 times"},{"priority":null,"date":"Mar 8 15:18:40","hostname":"127:0:ab::1","tag":"sshd","content":"unauthorized request"},{"priority":null,"date":"Mar 8 15:18:40","hostname":"server.example.com","tag":"sshd","content":"unauthorized request"},{"priority":null,"date":"Mar 8 15:18:40","hostname":"192.168.1.1","tag":"sshd","content":"unauthorized request"},{"priority":35,"date":"Mar 8 15:18:40","hostname":"server.example.com","tag":"sshd","content":"unauthorized request"},{"unparsable":"<7>unparsable line"}] From 8a6bbc55d258bcc9c9a89555bd122cb548c292cd Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Tue, 16 Aug 2022 10:01:27 -0700 Subject: [PATCH 077/116] fix for missing tag --- docs/parsers/syslog_bsd.md | 4 ++-- jc/parsers/syslog_bsd.py | 20 +++++++++++++++----- tests/fixtures/generic/syslog-3164.json | 2 +- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/docs/parsers/syslog_bsd.md b/docs/parsers/syslog_bsd.md index 824d62c63..7ffc01891 100644 --- a/docs/parsers/syslog_bsd.md +++ b/docs/parsers/syslog_bsd.md @@ -7,7 +7,7 @@ jc - JSON Convert Syslog RFC 3164 string parser This parser accepts a single syslog line string or multiple syslog lines separated by newlines. A warning message to `STDERR` will be printed if an -unparsable line is found. +unparsable line is found unless `--quiet` or `quiet=True` is used. Usage (cli): @@ -25,7 +25,7 @@ Schema: "priority": integer/null, "date": string, "hostname": string, - "tag": string, + "tag": string/null, "content": string, "unparsable": string, # [0] } diff --git a/jc/parsers/syslog_bsd.py b/jc/parsers/syslog_bsd.py index e6fe3f127..331fdb571 100644 --- a/jc/parsers/syslog_bsd.py +++ b/jc/parsers/syslog_bsd.py @@ -2,7 +2,7 @@ This parser accepts a single syslog line string or multiple syslog lines separated by newlines. A warning message to `STDERR` will be printed if an -unparsable line is found. +unparsable line is found unless `--quiet` or `quiet=True` is used. Usage (cli): @@ -20,7 +20,7 @@ "priority": integer/null, "date": string, "hostname": string, - "tag": string, + "tag": string/null, "content": string, "unparsable": string, # [0] } @@ -136,13 +136,23 @@ def parse( if syslog_match.group('priority'): priority = syslog_match.group('priority')[1:-1] + # check for missing tag + hostname = syslog_match.group('host') + tag = syslog_match.group('tag') + content = syslog_match.group('content') + if hostname: + if hostname.endswith(':'): + content = tag + content + tag = None + hostname = hostname[:-1] + syslog_dict = { 'priority': priority, 'date': syslog_match.group('date'), - 'hostname': syslog_match.group('host').rstrip(':'), + 'hostname': hostname, # 'raw_msg': syslog_match.group('msg'), - 'tag': syslog_match.group('tag'), - 'content': syslog_match.group('content').lstrip(' :').rstrip() + 'tag': tag, + 'content': content.lstrip(' :').rstrip() } else: diff --git a/tests/fixtures/generic/syslog-3164.json b/tests/fixtures/generic/syslog-3164.json index 667c4743f..4e8030114 100644 --- a/tests/fixtures/generic/syslog-3164.json +++ b/tests/fixtures/generic/syslog-3164.json @@ -1 +1 @@ -[{"priority":34,"date":"Oct 11 22:14:15","hostname":"mymachine","tag":"su","content":"'su root' failed for lonvick on /dev/pts/8"},{"priority":null,"date":"Oct 11 22:14:15","hostname":"mymachine","tag":"su","content":"'su root' failed for lonvick on /dev/pts/8"},{"priority":35,"date":"Oct 12 22:14:15","hostname":"client_machine","tag":"su","content":"'su root' failed for joe on /dev/pts/2"},{"priority":35,"date":"Mar 7 04:02:16","hostname":"avas","tag":"clamd","content":"[11165]: /var/amavis/amavis-20040307T033734-10329/parts/part-00003: Worm.Mydoom.F FOUND"},{"priority":null,"date":"Mar 7 04:05:55","hostname":"avas","tag":"clamd","content":"[11240]: /var/amavis/amavis-20040307T035901-10615/parts/part-00002: Worm.SomeFool.Gen-1 FOUND"},{"priority":5,"date":"Mar 7 09:00:51","hostname":"avas","tag":"clamd","content":"[27173]: SelfCheck: Database status OK."},{"priority":null,"date":"Mar 7 05:59:02","hostname":"avas","tag":"clamd","content":"[27173]: Database correctly reloaded (20400 viruses)"},{"priority":null,"date":"Mar 7 04:02:16","hostname":"avas","tag":"clamd","content":"[11165]: /var/amavis/amavis-20040307T033734-10329/parts/part-00003: Worm.Mydoom.F FOUND"},{"priority":null,"date":"Mar 7 04:05:55","hostname":"avas","tag":"clamd","content":"[11240]: /var/amavis/amavis-20040307T035901-10615/parts/part-00002: Worm.SomeFool.Gen-1 FOUND"},{"priority":null,"date":"Mar 7 09:00:51","hostname":"avas","tag":"clamd","content":"[27173]: SelfCheck: Database status OK."},{"priority":null,"date":"Mar 7 05:59:02","hostname":"avas","tag":"clamd","content":"[27173]: Database correctly reloaded (20400 viruses)"},{"priority":null,"date":"Mar 7 11:14:35","hostname":"avas","tag":"dccd","content":"[13284]: 21 requests/sec are too many from anonymous 205.201.1.56,2246"},{"priority":null,"date":"Mar 8 00:22:57","hostname":"avas","tag":"dccifd","content":"[9933]: write(MTA socket,4): Broken pipe"},{"priority":null,"date":"Mar 7 21:23:22","hostname":"avas","tag":"dccifd","content":"[6191]: missing message body"},{"priority":null,"date":"Mar 9 16:05:17","hostname":"avas","tag":"named","content":"[12045]: zone PLNet/IN: refresh: non-authoritative answer from master 10.0.0.253#53"},{"priority":null,"date":"Mar 10 00:38:16","hostname":"avas","tag":"dccifd","content":"[23069]: continue not asking DCC 17 seconds after failure"},{"priority":null,"date":"Mar 10 09:42:11","hostname":"avas","tag":"named","content":"client 127.0.0.1#55524: query: 23.68.27.142.sa-trusted.bondedsender.org IN TXT"},{"priority":null,"date":"Mar 9 03:48:07","hostname":"avas","tag":"dccd","content":"[145]: automatic dbclean; starting `dbclean -DPq -i 1189 -L info,local5.notice -L error,local5.err`"},{"priority":null,"date":"Mar 9 11:58:18","hostname":"avas","tag":"kernel","content":"i810_audio: Connection 0 with codec id 2"},{"priority":null,"date":"Mar 9 19:41:13","hostname":"avas","tag":"dccd","content":"[3004]: \"packet length 44 too small for REPORT\" sent to client 1 at 194.63.250.215,47577"},{"priority":null,"date":"Mar 8 09:01:07","hostname":"avas","tag":"sshd","content":"(pam_unix)[21839]: session opened for user tom by (uid=35567)"},{"priority":null,"date":"Mar 8 03:52:04","hostname":"avas","tag":"dccd","content":"[13284]: 1.2.32 database /home/dcc/dcc_db reopened with 997 MByte window"},{"priority":null,"date":"Mar 8 16:05:26","hostname":"avas","tag":"arpwatch","content":"listening on eth0"},{"priority":null,"date":"Mar 10 10:00:06","hostname":"avas","tag":"named","content":"[6986]: zone PLNet/IN: refresh: non-authoritative answer from master 192.75.26.21#53"},{"priority":null,"date":"Mar 10 10:00:10","hostname":"avas","tag":"named","content":"[6986]: client 127.0.0.1#55867: query: mail.canfor.ca IN MX"},{"priority":null,"date":"Mar 8 15:18:40","hostname":"avas","tag":"last","content":"message repeated 11 times"},{"priority":null,"date":"Mar 8 15:18:40","hostname":"127:0:ab::1","tag":"sshd","content":"unauthorized request"},{"priority":null,"date":"Mar 8 15:18:40","hostname":"server.example.com","tag":"sshd","content":"unauthorized request"},{"priority":null,"date":"Mar 8 15:18:40","hostname":"192.168.1.1","tag":"sshd","content":"unauthorized request"},{"priority":35,"date":"Mar 8 15:18:40","hostname":"server.example.com","tag":"sshd","content":"unauthorized request"},{"unparsable":"<7>unparsable line"}] +[{"priority":34,"date":"Oct 11 22:14:15","hostname":"mymachine","tag":"su","content":"'su root' failed for lonvick on /dev/pts/8"},{"priority":null,"date":"Oct 11 22:14:15","hostname":"mymachine","tag":"su","content":"'su root' failed for lonvick on /dev/pts/8"},{"priority":35,"date":"Oct 12 22:14:15","hostname":"client_machine","tag":"su","content":"'su root' failed for joe on /dev/pts/2"},{"priority":35,"date":"Mar 7 04:02:16","hostname":"avas","tag":"clamd","content":"[11165]: /var/amavis/amavis-20040307T033734-10329/parts/part-00003: Worm.Mydoom.F FOUND"},{"priority":null,"date":"Mar 7 04:05:55","hostname":"avas","tag":"clamd","content":"[11240]: /var/amavis/amavis-20040307T035901-10615/parts/part-00002: Worm.SomeFool.Gen-1 FOUND"},{"priority":5,"date":"Mar 7 09:00:51","hostname":"avas","tag":"clamd","content":"[27173]: SelfCheck: Database status OK."},{"priority":null,"date":"Mar 7 05:59:02","hostname":"avas","tag":"clamd","content":"[27173]: Database correctly reloaded (20400 viruses)"},{"priority":null,"date":"Mar 7 04:02:16","hostname":"avas","tag":"clamd","content":"[11165]: /var/amavis/amavis-20040307T033734-10329/parts/part-00003: Worm.Mydoom.F FOUND"},{"priority":null,"date":"Mar 7 04:05:55","hostname":"avas","tag":"clamd","content":"[11240]: /var/amavis/amavis-20040307T035901-10615/parts/part-00002: Worm.SomeFool.Gen-1 FOUND"},{"priority":null,"date":"Mar 7 09:00:51","hostname":"avas","tag":"clamd","content":"[27173]: SelfCheck: Database status OK."},{"priority":null,"date":"Mar 7 05:59:02","hostname":"avas","tag":"clamd","content":"[27173]: Database correctly reloaded (20400 viruses)"},{"priority":null,"date":"Mar 7 11:14:35","hostname":"avas","tag":"dccd","content":"[13284]: 21 requests/sec are too many from anonymous 205.201.1.56,2246"},{"priority":null,"date":"Mar 8 00:22:57","hostname":"avas","tag":"dccifd","content":"[9933]: write(MTA socket,4): Broken pipe"},{"priority":null,"date":"Mar 7 21:23:22","hostname":"avas","tag":"dccifd","content":"[6191]: missing message body"},{"priority":null,"date":"Mar 9 16:05:17","hostname":"avas","tag":"named","content":"[12045]: zone PLNet/IN: refresh: non-authoritative answer from master 10.0.0.253#53"},{"priority":null,"date":"Mar 10 00:38:16","hostname":"avas","tag":"dccifd","content":"[23069]: continue not asking DCC 17 seconds after failure"},{"priority":null,"date":"Mar 10 09:42:11","hostname":"avas","tag":"named","content":"client 127.0.0.1#55524: query: 23.68.27.142.sa-trusted.bondedsender.org IN TXT"},{"priority":null,"date":"Mar 9 03:48:07","hostname":"avas","tag":"dccd","content":"[145]: automatic dbclean; starting `dbclean -DPq -i 1189 -L info,local5.notice -L error,local5.err`"},{"priority":null,"date":"Mar 9 11:58:18","hostname":"avas","tag":"kernel","content":"i810_audio: Connection 0 with codec id 2"},{"priority":null,"date":"Mar 9 19:41:13","hostname":"avas","tag":"dccd","content":"[3004]: \"packet length 44 too small for REPORT\" sent to client 1 at 194.63.250.215,47577"},{"priority":null,"date":"Mar 8 09:01:07","hostname":"avas","tag":"sshd","content":"(pam_unix)[21839]: session opened for user tom by (uid=35567)"},{"priority":null,"date":"Mar 8 03:52:04","hostname":"avas","tag":"dccd","content":"[13284]: 1.2.32 database /home/dcc/dcc_db reopened with 997 MByte window"},{"priority":null,"date":"Mar 8 16:05:26","hostname":"avas","tag":"arpwatch","content":"listening on eth0"},{"priority":null,"date":"Mar 10 10:00:06","hostname":"avas","tag":"named","content":"[6986]: zone PLNet/IN: refresh: non-authoritative answer from master 192.75.26.21#53"},{"priority":null,"date":"Mar 10 10:00:10","hostname":"avas","tag":"named","content":"[6986]: client 127.0.0.1#55867: query: mail.canfor.ca IN MX"},{"priority":null,"date":"Mar 8 15:18:40","hostname":"avas","tag":null,"content":"last message repeated 11 times"},{"priority":null,"date":"Mar 8 15:18:40","hostname":"127:0:ab::1","tag":"sshd","content":"unauthorized request"},{"priority":null,"date":"Mar 8 15:18:40","hostname":"server.example.com","tag":"sshd","content":"unauthorized request"},{"priority":null,"date":"Mar 8 15:18:40","hostname":"192.168.1.1","tag":"sshd","content":"unauthorized request"},{"priority":35,"date":"Mar 8 15:18:40","hostname":"server.example.com","tag":"sshd","content":"unauthorized request"},{"unparsable":"<7>unparsable line"}] From f17d9ccbd6739d2d49027aa4e07c6dfc0530f38f Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Tue, 16 Aug 2022 10:01:31 -0700 Subject: [PATCH 078/116] doc update --- docs/parsers/syslog.md | 2 +- jc/parsers/syslog.py | 2 +- man/jc.1 | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/parsers/syslog.md b/docs/parsers/syslog.md index f5052d8b5..ca0eba981 100644 --- a/docs/parsers/syslog.md +++ b/docs/parsers/syslog.md @@ -7,7 +7,7 @@ jc - JSON Convert Syslog RFC 5424 string parser This parser accepts a single syslog line string or multiple syslog lines separated by newlines. A warning message to `STDERR` will be printed if an -unparsable line is found. +unparsable line is found unless `--quiet` or `quiet=True` is used. The `timestamp_epoch` calculated timestamp field is naive. (i.e. based on the local time of the system the parser is run on) diff --git a/jc/parsers/syslog.py b/jc/parsers/syslog.py index d2962804c..5915c71bd 100644 --- a/jc/parsers/syslog.py +++ b/jc/parsers/syslog.py @@ -2,7 +2,7 @@ This parser accepts a single syslog line string or multiple syslog lines separated by newlines. A warning message to `STDERR` will be printed if an -unparsable line is found. +unparsable line is found unless `--quiet` or `quiet=True` is used. The `timestamp_epoch` calculated timestamp field is naive. (i.e. based on the local time of the system the parser is run on) diff --git a/man/jc.1 b/man/jc.1 index f2269260c..4e8dcfadc 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -1,4 +1,4 @@ -.TH jc 1 2022-08-15 1.21.0 "JSON Convert" +.TH jc 1 2022-08-16 1.21.0 "JSON Convert" .SH NAME \fBjc\fP \- JSON Convert JSONifies the output of many CLI tools and file-types .SH SYNOPSIS From 167ea6dfcff8ee99122f3bb9480a27f4c28ff621 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Tue, 16 Aug 2022 14:36:00 -0700 Subject: [PATCH 079/116] add several fields based on user feedback --- jc/parsers/mdadm.py | 74 ++++++++++++++++--- .../generic/mdadm-examine-raid0-offline.json | 2 +- .../generic/mdadm-examine-raid0-ok.json | 2 +- .../generic/mdadm-examine-raid1-0-90-ok.json | 2 +- .../generic/mdadm-examine-raid1-checking.json | 2 +- .../generic/mdadm-examine-raid1-failfast.json | 2 +- .../generic/mdadm-examine-raid1-faulty1.json | 2 +- .../generic/mdadm-examine-raid1-faulty2.json | 2 +- .../mdadm-examine-raid1-moreflags.json | 2 +- .../generic/mdadm-examine-raid1-ok.json | 2 +- .../mdadm-examine-raid1-replacing.json | 2 +- .../generic/mdadm-examine-raid1-resync.json | 2 +- .../generic/mdadm-examine-raid1-spare.json | 2 +- .../generic/mdadm-examine-raid1-syncing.json | 2 +- tests/fixtures/generic/mdadm-examine.json | 2 +- .../fixtures/generic/mdadm-query-detail.json | 2 +- .../generic/mdadm-query-raid0-ok.json | 2 +- .../mdadm-query-raid1-failed-and-flags.json | 2 +- .../mdadm-query-raid1-faulty-and-removed.json | 2 +- .../mdadm-query-raid1-faulty-with-spare.json | 2 +- .../generic/mdadm-query-raid1-faulty.json | 2 +- .../generic/mdadm-query-raid1-ok-0-9.json | 2 +- .../mdadm-query-raid1-ok-failfast.json | 2 +- .../generic/mdadm-query-raid1-ok-spare.json | 2 +- .../generic/mdadm-query-raid1-ok.json | 2 +- .../mdadm-query-raid1-rebuild-failfast.json | 2 +- ...dadm-query-raid1-spare-writem-rebuild.json | 2 +- .../generic/mdadm-query-raid1-syncing.json | 2 +- 28 files changed, 92 insertions(+), 36 deletions(-) diff --git a/jc/parsers/mdadm.py b/jc/parsers/mdadm.py index 343974294..5c4fde129 100644 --- a/jc/parsers/mdadm.py +++ b/jc/parsers/mdadm.py @@ -35,6 +35,15 @@ "name_val": string, "uuid": string, "uuid_val": string, + "homehost": string, + "container": string, + "container_dev": string, + "container_member": integer, + "controller_guid": string, + "container_guid": string, + "seq": string, + "redundant_hdr": string, + "virtual_disks": integer, "creation_time": string, "creation_time_epoch": integer, # naive timestamp "raid_level": string, @@ -65,13 +74,21 @@ "checksum": string, "checksum_val": string, "checksum_state": string, - "events": integer, - "chunk_size": integer, + "events": string, + "events_num": integer, + "events_maj": integer, + "events_min": integer, + "chunk_size": string, + "chunk_size_num": integer, "device_role": string, "array_state": string, "array_state_list": [ string ], + "member_arrays": string, + "member_arrays_list": [ + string + ], "consistency_policy": string, "rebuild_status": string, "rebuild_status_percent": integer, @@ -86,6 +103,7 @@ "working_devices": integer, "failed_devices": integer, "spare_devices": integer, + "physical_disks": integer, "device_table": [ { "number": integer/null, @@ -100,6 +118,8 @@ ] } + Any fields unspecified above will be string type. + Examples: $ mdadm --query --detail /dev/md0 | jc --mdadm -p @@ -202,6 +222,7 @@ } """ from typing import Dict +import re import jc.utils from jc.parsers.universal import sparse_table_parse @@ -231,12 +252,12 @@ def _process(proc_data: Dict) -> Dict: Dictionary. Structured to conform to the schema. """ - int_list = {'array_size_num', 'used_dev_size_num', 'raid_devices', 'total_devices', - 'active_devices', 'working_devices', 'failed_devices', 'spare_devices', - 'events', 'number', 'major', 'minor', 'raid_device', 'avail_dev_size_num', + int_list = {'array_size_num', 'used_dev_size_num', 'raid_devices', 'total_devices', 'Number', + 'active_devices', 'working_devices', 'failed_devices', 'spare_devices', 'physical_disks', + 'number', 'major', 'minor', 'raid_device', 'avail_dev_size_num', 'virtual_disks', 'data_offset', 'super_offset', 'unused_space_before', 'unused_space_after', - 'chunk_size', 'preferred_minor', 'check_status_percent', 'resync_status_percent', - 'rebuild_status_percent'} + 'chunk_size_num', 'preferred_minor', 'check_status_percent', 'resync_status_percent', + 'rebuild_status_percent', 'events_num', 'events_maj', 'events_min', 'container_member'} array_state_map = { 'A': 'active', @@ -271,9 +292,29 @@ def _process(proc_data: Dict) -> Dict: if 'name' in proc_data: proc_data['name_val'] = proc_data['name'].split(maxsplit=1)[0] + if proc_data['name'].endswith(')'): + proc_data['homehost'] = proc_data['name'].rsplit(maxsplit=1)[1][0:-1] if 'uuid' in proc_data: proc_data['uuid_val'] = proc_data['uuid'].split(maxsplit=1)[0] + if proc_data['uuid'].endswith(')'): + proc_data['homehost'] = proc_data['uuid'].rsplit(maxsplit=1)[1][0:-1] + + if 'container' in proc_data: + if ', member ' in proc_data['container']: + proc_data['container_dev'] = proc_data['container'].split(',')[0] + proc_data['container_member'] = proc_data['container'].rsplit(maxsplit=1)[-1] + + if 'chunk_size' in proc_data: + proc_data['chunk_size_num'] = proc_data['chunk_size'] + + if 'events' in proc_data: + if '.' in proc_data['events']: + events_maj, events_min = proc_data['events'].split('.', maxsplit=1) + proc_data['events_maj'] = events_maj + proc_data['events_min'] = events_min + else: + proc_data['events_num'] = proc_data['events'] if 'checksum' in proc_data: proc_data['checksum_val'] = proc_data['checksum'].split(maxsplit=1)[0] @@ -286,6 +327,9 @@ def _process(proc_data: Dict) -> Dict: if 'flags' in proc_data: proc_data['flag_list'] = proc_data['flags'].split() + if 'member_arrays' in proc_data: + proc_data['member_arrays_list'] = proc_data['member_arrays'].split() + if 'array_state' in proc_data: array_state_list = [] array_state_text = proc_data['array_state'].split(maxsplit=1)[0] @@ -313,7 +357,7 @@ def _process(proc_data: Dict) -> Dict: dt = jc.utils.timestamp(proc_data['update_time'], format_hint=(1000,)) proc_data['update_time_epoch'] = dt.naive - # convert ints + # convert ints/floats for key in proc_data: if key in int_list: proc_data[key] = jc .utils.convert_to_int(proc_data[key]) @@ -368,7 +412,9 @@ def parse( # key/value lines if ' : ' in line: key, value = line.split(' : ', maxsplit=1) - key = key.strip().lower().replace(' ', '_') + key = key.strip().lower() + key = re.sub(r'[^a-z0-9]', '_', key) + key = key.strip('_') value = value.strip() raw_output[key] = value continue @@ -379,6 +425,16 @@ def parse( device_table = True continue + elif ' Number Major Minor RaidDevice' in line: + device_table_list.append(' number major minor RaidDevice device') + device_table = True + continue + + elif ' Number RefNo Size Device Type/State' in line: + device_table_list.append(' Number RefNo Size Device Type/State') + device_table = True + continue + # device table lines if device_table == True: device_table_list.append(line) diff --git a/tests/fixtures/generic/mdadm-examine-raid0-offline.json b/tests/fixtures/generic/mdadm-examine-raid0-offline.json index 1ad0d50af..685d9686e 100644 --- a/tests/fixtures/generic/mdadm-examine-raid0-offline.json +++ b/tests/fixtures/generic/mdadm-examine-raid0-offline.json @@ -1 +1 @@ -{"device":"/dev/vda1","magic":"a92b4efc","version":"1.2","feature_map":"0x0","array_uuid":"7e81a856:abb9c1c2:4b71237a:9778cc66","name":"sysrescue:0 (local to host sysrescue)","creation_time":"Sun Aug 7 20:56:52 2022","raid_level":"raid0","raid_devices":2,"avail_dev_size":"200704 sectors (98.00 MiB 102.76 MB)","data_offset":4096,"super_offset":8,"unused_space":"before=4016 sectors, after=0 sectors","state":"clean","device_uuid":"a49d96f1:8e86ca03:5ff499ec:8cdd42bb","update_time":"Sun Aug 7 20:56:52 2022","bad_block_log":"512 entries available at offset 8 sectors","checksum":"eeb6cbe3 - correct","events":0,"chunk_size":512,"device_role":"Active device 0","array_state":"AA ('A' == active, '.' == missing, 'R' == replacing)","avail_dev_size_num":200704,"unused_space_before":4016,"unused_space_after":0,"name_val":"sysrescue:0","checksum_val":"eeb6cbe3","checksum_state":"correct","state_list":["clean"],"array_state_list":["active","active"],"creation_time_epoch":1659931012,"update_time_epoch":1659931012} +{"device":"/dev/vda1","magic":"a92b4efc","version":"1.2","feature_map":"0x0","array_uuid":"7e81a856:abb9c1c2:4b71237a:9778cc66","name":"sysrescue:0 (local to host sysrescue)","creation_time":"Sun Aug 7 20:56:52 2022","raid_level":"raid0","raid_devices":2,"avail_dev_size":"200704 sectors (98.00 MiB 102.76 MB)","data_offset":4096,"super_offset":8,"unused_space":"before=4016 sectors, after=0 sectors","state":"clean","device_uuid":"a49d96f1:8e86ca03:5ff499ec:8cdd42bb","update_time":"Sun Aug 7 20:56:52 2022","bad_block_log":"512 entries available at offset 8 sectors","checksum":"eeb6cbe3 - correct","events":"0","chunk_size":"512K","device_role":"Active device 0","array_state":"AA ('A' == active, '.' == missing, 'R' == replacing)","avail_dev_size_num":200704,"unused_space_before":4016,"unused_space_after":0,"name_val":"sysrescue:0","homehost":"sysrescue","chunk_size_num":512,"events_num":0,"checksum_val":"eeb6cbe3","checksum_state":"correct","state_list":["clean"],"array_state_list":["active","active"],"creation_time_epoch":1659931012,"update_time_epoch":1659931012} diff --git a/tests/fixtures/generic/mdadm-examine-raid0-ok.json b/tests/fixtures/generic/mdadm-examine-raid0-ok.json index 1ad0d50af..685d9686e 100644 --- a/tests/fixtures/generic/mdadm-examine-raid0-ok.json +++ b/tests/fixtures/generic/mdadm-examine-raid0-ok.json @@ -1 +1 @@ -{"device":"/dev/vda1","magic":"a92b4efc","version":"1.2","feature_map":"0x0","array_uuid":"7e81a856:abb9c1c2:4b71237a:9778cc66","name":"sysrescue:0 (local to host sysrescue)","creation_time":"Sun Aug 7 20:56:52 2022","raid_level":"raid0","raid_devices":2,"avail_dev_size":"200704 sectors (98.00 MiB 102.76 MB)","data_offset":4096,"super_offset":8,"unused_space":"before=4016 sectors, after=0 sectors","state":"clean","device_uuid":"a49d96f1:8e86ca03:5ff499ec:8cdd42bb","update_time":"Sun Aug 7 20:56:52 2022","bad_block_log":"512 entries available at offset 8 sectors","checksum":"eeb6cbe3 - correct","events":0,"chunk_size":512,"device_role":"Active device 0","array_state":"AA ('A' == active, '.' == missing, 'R' == replacing)","avail_dev_size_num":200704,"unused_space_before":4016,"unused_space_after":0,"name_val":"sysrescue:0","checksum_val":"eeb6cbe3","checksum_state":"correct","state_list":["clean"],"array_state_list":["active","active"],"creation_time_epoch":1659931012,"update_time_epoch":1659931012} +{"device":"/dev/vda1","magic":"a92b4efc","version":"1.2","feature_map":"0x0","array_uuid":"7e81a856:abb9c1c2:4b71237a:9778cc66","name":"sysrescue:0 (local to host sysrescue)","creation_time":"Sun Aug 7 20:56:52 2022","raid_level":"raid0","raid_devices":2,"avail_dev_size":"200704 sectors (98.00 MiB 102.76 MB)","data_offset":4096,"super_offset":8,"unused_space":"before=4016 sectors, after=0 sectors","state":"clean","device_uuid":"a49d96f1:8e86ca03:5ff499ec:8cdd42bb","update_time":"Sun Aug 7 20:56:52 2022","bad_block_log":"512 entries available at offset 8 sectors","checksum":"eeb6cbe3 - correct","events":"0","chunk_size":"512K","device_role":"Active device 0","array_state":"AA ('A' == active, '.' == missing, 'R' == replacing)","avail_dev_size_num":200704,"unused_space_before":4016,"unused_space_after":0,"name_val":"sysrescue:0","homehost":"sysrescue","chunk_size_num":512,"events_num":0,"checksum_val":"eeb6cbe3","checksum_state":"correct","state_list":["clean"],"array_state_list":["active","active"],"creation_time_epoch":1659931012,"update_time_epoch":1659931012} diff --git a/tests/fixtures/generic/mdadm-examine-raid1-0-90-ok.json b/tests/fixtures/generic/mdadm-examine-raid1-0-90-ok.json index 4f538f22d..31f8aeb0a 100644 --- a/tests/fixtures/generic/mdadm-examine-raid1-0-90-ok.json +++ b/tests/fixtures/generic/mdadm-examine-raid1-0-90-ok.json @@ -1 +1 @@ -{"device":"/dev/vda1","magic":"a92b4efc","version":"0.90.00","uuid":"3f720601:29af5283:96fc04a8:108a4af7 (local to host sysrescue)","creation_time":"Sun Aug 7 21:52:56 2022","raid_level":"raid1","used_dev_size":"102336 (99.94 MiB 104.79 MB)","array_size":"102336 (99.94 MiB 104.79 MB)","raid_devices":2,"total_devices":2,"preferred_minor":0,"update_time":"Sun Aug 7 21:54:35 2022","state":"clean","active_devices":2,"working_devices":2,"failed_devices":0,"spare_devices":0,"checksum":"7fb4bb0e - correct","events":18,"device_table":[{"number":0,"major":254,"minor":1,"state":["active","sync"],"device":"/dev/vda1","raid_device":0},{"number":0,"major":254,"minor":1,"state":["active","sync"],"device":"/dev/vda1","raid_device":0},{"number":11,"major":254,"minor":2,"state":["active","sync"],"device":"/dev/vda2","raid_device":1}],"array_size_num":102336,"used_dev_size_num":102336,"uuid_val":"3f720601:29af5283:96fc04a8:108a4af7","checksum_val":"7fb4bb0e","checksum_state":"correct","state_list":["clean"],"creation_time_epoch":1659934376,"update_time_epoch":1659934475} +{"device":"/dev/vda1","magic":"a92b4efc","version":"0.90.00","uuid":"3f720601:29af5283:96fc04a8:108a4af7 (local to host sysrescue)","creation_time":"Sun Aug 7 21:52:56 2022","raid_level":"raid1","used_dev_size":"102336 (99.94 MiB 104.79 MB)","array_size":"102336 (99.94 MiB 104.79 MB)","raid_devices":2,"total_devices":2,"preferred_minor":0,"update_time":"Sun Aug 7 21:54:35 2022","state":"clean","active_devices":2,"working_devices":2,"failed_devices":0,"spare_devices":0,"checksum":"7fb4bb0e - correct","events":"18","device_table":[{"number":0,"major":254,"minor":1,"state":["active","sync"],"device":"/dev/vda1","raid_device":0},{"number":0,"major":254,"minor":1,"state":["active","sync"],"device":"/dev/vda1","raid_device":0},{"number":11,"major":254,"minor":2,"state":["active","sync"],"device":"/dev/vda2","raid_device":1}],"array_size_num":102336,"used_dev_size_num":102336,"uuid_val":"3f720601:29af5283:96fc04a8:108a4af7","homehost":"sysrescue","events_num":18,"checksum_val":"7fb4bb0e","checksum_state":"correct","state_list":["clean"],"creation_time_epoch":1659934376,"update_time_epoch":1659934475} diff --git a/tests/fixtures/generic/mdadm-examine-raid1-checking.json b/tests/fixtures/generic/mdadm-examine-raid1-checking.json index 9be39db60..37442cffe 100644 --- a/tests/fixtures/generic/mdadm-examine-raid1-checking.json +++ b/tests/fixtures/generic/mdadm-examine-raid1-checking.json @@ -1 +1 @@ -{"device":"/dev/md126","version":"1.2","creation_time":"Sun Aug 7 21:15:49 2022","raid_level":"raid1","array_size":"101376 (99.00 MiB 103.81 MB)","used_dev_size":"101376 (99.00 MiB 103.81 MB)","raid_devices":2,"total_devices":2,"persistence":"Superblock is persistent","update_time":"Sun Aug 7 21:23:59 2022","state":"clean, checking","active_devices":2,"working_devices":2,"failed_devices":0,"spare_devices":0,"consistency_policy":"resync","check_status":"21% complete","name":"sysrescue:126 (local to host sysrescue)","uuid":"e054553a:fbccdcfe:0ae80bc8:9379377a","events":18,"device_table":[{"number":0,"major":254,"minor":1,"state":["active","sync"],"device":"/dev/vda1","raid_device":0},{"number":1,"major":254,"minor":2,"state":["active","sync"],"device":"/dev/vda2","raid_device":1}],"array_size_num":101376,"used_dev_size_num":101376,"name_val":"sysrescue:126","uuid_val":"e054553a:fbccdcfe:0ae80bc8:9379377a","state_list":["clean","checking"],"check_status_percent":21,"creation_time_epoch":1659932149,"update_time_epoch":1659932639} +{"device":"/dev/md126","version":"1.2","creation_time":"Sun Aug 7 21:15:49 2022","raid_level":"raid1","array_size":"101376 (99.00 MiB 103.81 MB)","used_dev_size":"101376 (99.00 MiB 103.81 MB)","raid_devices":2,"total_devices":2,"persistence":"Superblock is persistent","update_time":"Sun Aug 7 21:23:59 2022","state":"clean, checking","active_devices":2,"working_devices":2,"failed_devices":0,"spare_devices":0,"consistency_policy":"resync","check_status":"21% complete","name":"sysrescue:126 (local to host sysrescue)","uuid":"e054553a:fbccdcfe:0ae80bc8:9379377a","events":"18","device_table":[{"number":0,"major":254,"minor":1,"state":["active","sync"],"device":"/dev/vda1","raid_device":0},{"number":1,"major":254,"minor":2,"state":["active","sync"],"device":"/dev/vda2","raid_device":1}],"array_size_num":101376,"used_dev_size_num":101376,"name_val":"sysrescue:126","homehost":"sysrescue","uuid_val":"e054553a:fbccdcfe:0ae80bc8:9379377a","events_num":18,"state_list":["clean","checking"],"check_status_percent":21,"creation_time_epoch":1659932149,"update_time_epoch":1659932639} diff --git a/tests/fixtures/generic/mdadm-examine-raid1-failfast.json b/tests/fixtures/generic/mdadm-examine-raid1-failfast.json index d6ed6a454..9a9252022 100644 --- a/tests/fixtures/generic/mdadm-examine-raid1-failfast.json +++ b/tests/fixtures/generic/mdadm-examine-raid1-failfast.json @@ -1 +1 @@ -{"device":"/dev/vda2","magic":"a92b4efc","version":"1.2","feature_map":"0x0","array_uuid":"e054553a:fbccdcfe:0ae80bc8:9379377a","name":"sysrescue:126 (local to host sysrescue)","creation_time":"Sun Aug 7 21:15:49 2022","raid_level":"raid1","raid_devices":2,"avail_dev_size":"202752 sectors (99.00 MiB 103.81 MB)","array_size":"101376 KiB (99.00 MiB 103.81 MB)","data_offset":2048,"super_offset":8,"unused_space":"before=1968 sectors, after=0 sectors","state":"clean","device_uuid":"140a340b:69d5759c:b87ce67c:6579ce22","flags":"failfast","update_time":"Sun Aug 7 21:37:34 2022","bad_block_log":"512 entries available at offset 16 sectors","checksum":"4bf42c40 - correct","events":51,"device_role":"Active device 1","array_state":"AA ('A' == active, '.' == missing, 'R' == replacing)","array_size_num":101376,"avail_dev_size_num":202752,"unused_space_before":1968,"unused_space_after":0,"name_val":"sysrescue:126","checksum_val":"4bf42c40","checksum_state":"correct","state_list":["clean"],"flag_list":["failfast"],"array_state_list":["active","active"],"creation_time_epoch":1659932149,"update_time_epoch":1659933454} +{"device":"/dev/vda2","magic":"a92b4efc","version":"1.2","feature_map":"0x0","array_uuid":"e054553a:fbccdcfe:0ae80bc8:9379377a","name":"sysrescue:126 (local to host sysrescue)","creation_time":"Sun Aug 7 21:15:49 2022","raid_level":"raid1","raid_devices":2,"avail_dev_size":"202752 sectors (99.00 MiB 103.81 MB)","array_size":"101376 KiB (99.00 MiB 103.81 MB)","data_offset":2048,"super_offset":8,"unused_space":"before=1968 sectors, after=0 sectors","state":"clean","device_uuid":"140a340b:69d5759c:b87ce67c:6579ce22","flags":"failfast","update_time":"Sun Aug 7 21:37:34 2022","bad_block_log":"512 entries available at offset 16 sectors","checksum":"4bf42c40 - correct","events":"51","device_role":"Active device 1","array_state":"AA ('A' == active, '.' == missing, 'R' == replacing)","array_size_num":101376,"avail_dev_size_num":202752,"unused_space_before":1968,"unused_space_after":0,"name_val":"sysrescue:126","homehost":"sysrescue","events_num":51,"checksum_val":"4bf42c40","checksum_state":"correct","state_list":["clean"],"flag_list":["failfast"],"array_state_list":["active","active"],"creation_time_epoch":1659932149,"update_time_epoch":1659933454} diff --git a/tests/fixtures/generic/mdadm-examine-raid1-faulty1.json b/tests/fixtures/generic/mdadm-examine-raid1-faulty1.json index b98cccd53..3ad9307e4 100644 --- a/tests/fixtures/generic/mdadm-examine-raid1-faulty1.json +++ b/tests/fixtures/generic/mdadm-examine-raid1-faulty1.json @@ -1 +1 @@ -{"device":"/dev/vda1","magic":"a92b4efc","version":"1.2","feature_map":"0x0","array_uuid":"e054553a:fbccdcfe:0ae80bc8:9379377a","name":"sysrescue:126 (local to host sysrescue)","creation_time":"Sun Aug 7 21:15:49 2022","raid_level":"raid1","raid_devices":2,"avail_dev_size":"202752 sectors (99.00 MiB 103.81 MB)","array_size":"101376 KiB (99.00 MiB 103.81 MB)","data_offset":2048,"super_offset":8,"unused_space":"before=1968 sectors, after=0 sectors","state":"clean","device_uuid":"1ec6a01d:e96d7f47:59970cf2:f0c056d2","update_time":"Sun Aug 7 21:28:34 2022","bad_block_log":"512 entries available at offset 16 sectors","checksum":"2e18e0bb - correct","events":25,"device_role":"Active device 0","array_state":"A. ('A' == active, '.' == missing, 'R' == replacing)","array_size_num":101376,"avail_dev_size_num":202752,"unused_space_before":1968,"unused_space_after":0,"name_val":"sysrescue:126","checksum_val":"2e18e0bb","checksum_state":"correct","state_list":["clean"],"array_state_list":["active","missing"],"creation_time_epoch":1659932149,"update_time_epoch":1659932914} +{"device":"/dev/vda1","magic":"a92b4efc","version":"1.2","feature_map":"0x0","array_uuid":"e054553a:fbccdcfe:0ae80bc8:9379377a","name":"sysrescue:126 (local to host sysrescue)","creation_time":"Sun Aug 7 21:15:49 2022","raid_level":"raid1","raid_devices":2,"avail_dev_size":"202752 sectors (99.00 MiB 103.81 MB)","array_size":"101376 KiB (99.00 MiB 103.81 MB)","data_offset":2048,"super_offset":8,"unused_space":"before=1968 sectors, after=0 sectors","state":"clean","device_uuid":"1ec6a01d:e96d7f47:59970cf2:f0c056d2","update_time":"Sun Aug 7 21:28:34 2022","bad_block_log":"512 entries available at offset 16 sectors","checksum":"2e18e0bb - correct","events":"25","device_role":"Active device 0","array_state":"A. ('A' == active, '.' == missing, 'R' == replacing)","array_size_num":101376,"avail_dev_size_num":202752,"unused_space_before":1968,"unused_space_after":0,"name_val":"sysrescue:126","homehost":"sysrescue","events_num":25,"checksum_val":"2e18e0bb","checksum_state":"correct","state_list":["clean"],"array_state_list":["active","missing"],"creation_time_epoch":1659932149,"update_time_epoch":1659932914} diff --git a/tests/fixtures/generic/mdadm-examine-raid1-faulty2.json b/tests/fixtures/generic/mdadm-examine-raid1-faulty2.json index 95ac28507..0c4cb080a 100644 --- a/tests/fixtures/generic/mdadm-examine-raid1-faulty2.json +++ b/tests/fixtures/generic/mdadm-examine-raid1-faulty2.json @@ -1 +1 @@ -{"device":"/dev/vda2","magic":"a92b4efc","version":"1.2","feature_map":"0x0","array_uuid":"e054553a:fbccdcfe:0ae80bc8:9379377a","name":"sysrescue:126 (local to host sysrescue)","creation_time":"Sun Aug 7 21:15:49 2022","raid_level":"raid1","raid_devices":2,"avail_dev_size":"202752 sectors (99.00 MiB 103.81 MB)","array_size":"101376 KiB (99.00 MiB 103.81 MB)","data_offset":2048,"super_offset":8,"unused_space":"before=1968 sectors, after=0 sectors","state":"clean","device_uuid":"66e4da05:9cc62990:0620d976:579c0948","update_time":"Sun Aug 7 21:27:02 2022","bad_block_log":"512 entries available at offset 16 sectors","checksum":"597fbb6b - correct","events":23,"device_role":"Active device 1","array_state":"AA ('A' == active, '.' == missing, 'R' == replacing)","array_size_num":101376,"avail_dev_size_num":202752,"unused_space_before":1968,"unused_space_after":0,"name_val":"sysrescue:126","checksum_val":"597fbb6b","checksum_state":"correct","state_list":["clean"],"array_state_list":["active","active"],"creation_time_epoch":1659932149,"update_time_epoch":1659932822} +{"device":"/dev/vda2","magic":"a92b4efc","version":"1.2","feature_map":"0x0","array_uuid":"e054553a:fbccdcfe:0ae80bc8:9379377a","name":"sysrescue:126 (local to host sysrescue)","creation_time":"Sun Aug 7 21:15:49 2022","raid_level":"raid1","raid_devices":2,"avail_dev_size":"202752 sectors (99.00 MiB 103.81 MB)","array_size":"101376 KiB (99.00 MiB 103.81 MB)","data_offset":2048,"super_offset":8,"unused_space":"before=1968 sectors, after=0 sectors","state":"clean","device_uuid":"66e4da05:9cc62990:0620d976:579c0948","update_time":"Sun Aug 7 21:27:02 2022","bad_block_log":"512 entries available at offset 16 sectors","checksum":"597fbb6b - correct","events":"23","device_role":"Active device 1","array_state":"AA ('A' == active, '.' == missing, 'R' == replacing)","array_size_num":101376,"avail_dev_size_num":202752,"unused_space_before":1968,"unused_space_after":0,"name_val":"sysrescue:126","homehost":"sysrescue","events_num":23,"checksum_val":"597fbb6b","checksum_state":"correct","state_list":["clean"],"array_state_list":["active","active"],"creation_time_epoch":1659932149,"update_time_epoch":1659932822} diff --git a/tests/fixtures/generic/mdadm-examine-raid1-moreflags.json b/tests/fixtures/generic/mdadm-examine-raid1-moreflags.json index c17470281..ca946587e 100644 --- a/tests/fixtures/generic/mdadm-examine-raid1-moreflags.json +++ b/tests/fixtures/generic/mdadm-examine-raid1-moreflags.json @@ -1 +1 @@ -{"device":"/dev/vda2","magic":"a92b4efc","version":"1.2","feature_map":"0x0","array_uuid":"e054553a:fbccdcfe:0ae80bc8:9379377a","name":"sysrescue:126 (local to host sysrescue)","creation_time":"Sun Aug 7 21:15:49 2022","raid_level":"raid1","raid_devices":2,"avail_dev_size":"202752 sectors (99.00 MiB 103.81 MB)","array_size":"101376 KiB (99.00 MiB 103.81 MB)","data_offset":2048,"super_offset":8,"unused_space":"before=1968 sectors, after=0 sectors","state":"clean","device_uuid":"cc336e3b:67a49dea:480135ed:9e1280cd","flags":"write-mostly failfast","update_time":"Sun Aug 7 21:48:58 2022","bad_block_log":"512 entries available at offset 16 sectors","checksum":"e5554588 - correct","events":78,"device_role":"Active device 1","array_state":"AA ('A' == active, '.' == missing, 'R' == replacing)","array_size_num":101376,"avail_dev_size_num":202752,"unused_space_before":1968,"unused_space_after":0,"name_val":"sysrescue:126","checksum_val":"e5554588","checksum_state":"correct","state_list":["clean"],"flag_list":["write-mostly","failfast"],"array_state_list":["active","active"],"creation_time_epoch":1659932149,"update_time_epoch":1659934138} +{"device":"/dev/vda2","magic":"a92b4efc","version":"1.2","feature_map":"0x0","array_uuid":"e054553a:fbccdcfe:0ae80bc8:9379377a","name":"sysrescue:126 (local to host sysrescue)","creation_time":"Sun Aug 7 21:15:49 2022","raid_level":"raid1","raid_devices":2,"avail_dev_size":"202752 sectors (99.00 MiB 103.81 MB)","array_size":"101376 KiB (99.00 MiB 103.81 MB)","data_offset":2048,"super_offset":8,"unused_space":"before=1968 sectors, after=0 sectors","state":"clean","device_uuid":"cc336e3b:67a49dea:480135ed:9e1280cd","flags":"write-mostly failfast","update_time":"Sun Aug 7 21:48:58 2022","bad_block_log":"512 entries available at offset 16 sectors","checksum":"e5554588 - correct","events":"78","device_role":"Active device 1","array_state":"AA ('A' == active, '.' == missing, 'R' == replacing)","array_size_num":101376,"avail_dev_size_num":202752,"unused_space_before":1968,"unused_space_after":0,"name_val":"sysrescue:126","homehost":"sysrescue","events_num":78,"checksum_val":"e5554588","checksum_state":"correct","state_list":["clean"],"flag_list":["write-mostly","failfast"],"array_state_list":["active","active"],"creation_time_epoch":1659932149,"update_time_epoch":1659934138} diff --git a/tests/fixtures/generic/mdadm-examine-raid1-ok.json b/tests/fixtures/generic/mdadm-examine-raid1-ok.json index 3a271f974..54c8df152 100644 --- a/tests/fixtures/generic/mdadm-examine-raid1-ok.json +++ b/tests/fixtures/generic/mdadm-examine-raid1-ok.json @@ -1 +1 @@ -{"device":"/dev/vda1","magic":"a92b4efc","version":"1.2","feature_map":"0x0","array_uuid":"e054553a:fbccdcfe:0ae80bc8:9379377a","name":"sysrescue:126 (local to host sysrescue)","creation_time":"Sun Aug 7 21:15:49 2022","raid_level":"raid1","raid_devices":2,"avail_dev_size":"202752 sectors (99.00 MiB 103.81 MB)","array_size":"101376 KiB (99.00 MiB 103.81 MB)","data_offset":2048,"super_offset":8,"unused_space":"before=1968 sectors, after=0 sectors","state":"clean","device_uuid":"1ec6a01d:e96d7f47:59970cf2:f0c056d2","update_time":"Sun Aug 7 21:20:08 2022","bad_block_log":"512 entries available at offset 16 sectors","checksum":"2e1bdeb8 - correct","events":17,"device_role":"Active device 0","array_state":"AA ('A' == active, '.' == missing, 'R' == replacing)","array_size_num":101376,"avail_dev_size_num":202752,"unused_space_before":1968,"unused_space_after":0,"name_val":"sysrescue:126","checksum_val":"2e1bdeb8","checksum_state":"correct","state_list":["clean"],"array_state_list":["active","active"],"creation_time_epoch":1659932149,"update_time_epoch":1659932408} +{"device":"/dev/vda1","magic":"a92b4efc","version":"1.2","feature_map":"0x0","array_uuid":"e054553a:fbccdcfe:0ae80bc8:9379377a","name":"sysrescue:126 (local to host sysrescue)","creation_time":"Sun Aug 7 21:15:49 2022","raid_level":"raid1","raid_devices":2,"avail_dev_size":"202752 sectors (99.00 MiB 103.81 MB)","array_size":"101376 KiB (99.00 MiB 103.81 MB)","data_offset":2048,"super_offset":8,"unused_space":"before=1968 sectors, after=0 sectors","state":"clean","device_uuid":"1ec6a01d:e96d7f47:59970cf2:f0c056d2","update_time":"Sun Aug 7 21:20:08 2022","bad_block_log":"512 entries available at offset 16 sectors","checksum":"2e1bdeb8 - correct","events":"17","device_role":"Active device 0","array_state":"AA ('A' == active, '.' == missing, 'R' == replacing)","array_size_num":101376,"avail_dev_size_num":202752,"unused_space_before":1968,"unused_space_after":0,"name_val":"sysrescue:126","homehost":"sysrescue","events_num":17,"checksum_val":"2e1bdeb8","checksum_state":"correct","state_list":["clean"],"array_state_list":["active","active"],"creation_time_epoch":1659932149,"update_time_epoch":1659932408} diff --git a/tests/fixtures/generic/mdadm-examine-raid1-replacing.json b/tests/fixtures/generic/mdadm-examine-raid1-replacing.json index 92f496298..09cf1b7a9 100644 --- a/tests/fixtures/generic/mdadm-examine-raid1-replacing.json +++ b/tests/fixtures/generic/mdadm-examine-raid1-replacing.json @@ -1 +1 @@ -{"device":"/dev/vda2","magic":"a92b4efc","version":"1.2","feature_map":"0x12","array_uuid":"e054553a:fbccdcfe:0ae80bc8:9379377a","name":"sysrescue:126 (local to host sysrescue)","creation_time":"Sun Aug 7 21:15:49 2022","raid_level":"raid1","raid_devices":2,"avail_dev_size":"202752 sectors (99.00 MiB 103.81 MB)","array_size":"101376 KiB (99.00 MiB 103.81 MB)","data_offset":2048,"super_offset":8,"recovery_offset":"115200 sectors","unused_space":"before=1968 sectors, after=0 sectors","state":"clean","device_uuid":"cc336e3b:67a49dea:480135ed:9e1280cd","flags":"write-mostly failfast","update_time":"Sun Aug 7 21:48:15 2022","bad_block_log":"512 entries available at offset 16 sectors","checksum":"e55a076b - correct","events":75,"device_role":"Replacement device 1","array_state":"AR ('A' == active, '.' == missing, 'R' == replacing)","array_size_num":101376,"avail_dev_size_num":202752,"unused_space_before":1968,"unused_space_after":0,"name_val":"sysrescue:126","checksum_val":"e55a076b","checksum_state":"correct","state_list":["clean"],"flag_list":["write-mostly","failfast"],"array_state_list":["active","replacing"],"creation_time_epoch":1659932149,"update_time_epoch":1659934095} +{"device":"/dev/vda2","magic":"a92b4efc","version":"1.2","feature_map":"0x12","array_uuid":"e054553a:fbccdcfe:0ae80bc8:9379377a","name":"sysrescue:126 (local to host sysrescue)","creation_time":"Sun Aug 7 21:15:49 2022","raid_level":"raid1","raid_devices":2,"avail_dev_size":"202752 sectors (99.00 MiB 103.81 MB)","array_size":"101376 KiB (99.00 MiB 103.81 MB)","data_offset":2048,"super_offset":8,"recovery_offset":"115200 sectors","unused_space":"before=1968 sectors, after=0 sectors","state":"clean","device_uuid":"cc336e3b:67a49dea:480135ed:9e1280cd","flags":"write-mostly failfast","update_time":"Sun Aug 7 21:48:15 2022","bad_block_log":"512 entries available at offset 16 sectors","checksum":"e55a076b - correct","events":"75","device_role":"Replacement device 1","array_state":"AR ('A' == active, '.' == missing, 'R' == replacing)","array_size_num":101376,"avail_dev_size_num":202752,"unused_space_before":1968,"unused_space_after":0,"name_val":"sysrescue:126","homehost":"sysrescue","events_num":75,"checksum_val":"e55a076b","checksum_state":"correct","state_list":["clean"],"flag_list":["write-mostly","failfast"],"array_state_list":["active","replacing"],"creation_time_epoch":1659932149,"update_time_epoch":1659934095} diff --git a/tests/fixtures/generic/mdadm-examine-raid1-resync.json b/tests/fixtures/generic/mdadm-examine-raid1-resync.json index 69c9fe0c2..d22d97874 100644 --- a/tests/fixtures/generic/mdadm-examine-raid1-resync.json +++ b/tests/fixtures/generic/mdadm-examine-raid1-resync.json @@ -1 +1 @@ -{"device":"/dev/md126","version":"1.2","creation_time":"Sun Aug 7 21:15:49 2022","raid_level":"raid1","array_size":"101376 (99.00 MiB 103.81 MB)","used_dev_size":"101376 (99.00 MiB 103.81 MB)","raid_devices":2,"total_devices":2,"persistence":"Superblock is persistent","update_time":"Sun Aug 7 21:25:52 2022","state":"clean, resyncing","active_devices":2,"working_devices":2,"failed_devices":0,"spare_devices":0,"consistency_policy":"resync","resync_status":"27% complete","name":"sysrescue:126 (local to host sysrescue)","uuid":"e054553a:fbccdcfe:0ae80bc8:9379377a","events":20,"device_table":[{"number":0,"major":254,"minor":1,"state":["active","sync"],"device":"/dev/vda1","raid_device":0},{"number":1,"major":254,"minor":2,"state":["active","sync"],"device":"/dev/vda2","raid_device":1}],"array_size_num":101376,"used_dev_size_num":101376,"name_val":"sysrescue:126","uuid_val":"e054553a:fbccdcfe:0ae80bc8:9379377a","state_list":["clean","resyncing"],"resync_status_percent":27,"creation_time_epoch":1659932149,"update_time_epoch":1659932752} +{"device":"/dev/md126","version":"1.2","creation_time":"Sun Aug 7 21:15:49 2022","raid_level":"raid1","array_size":"101376 (99.00 MiB 103.81 MB)","used_dev_size":"101376 (99.00 MiB 103.81 MB)","raid_devices":2,"total_devices":2,"persistence":"Superblock is persistent","update_time":"Sun Aug 7 21:25:52 2022","state":"clean, resyncing","active_devices":2,"working_devices":2,"failed_devices":0,"spare_devices":0,"consistency_policy":"resync","resync_status":"27% complete","name":"sysrescue:126 (local to host sysrescue)","uuid":"e054553a:fbccdcfe:0ae80bc8:9379377a","events":"20","device_table":[{"number":0,"major":254,"minor":1,"state":["active","sync"],"device":"/dev/vda1","raid_device":0},{"number":1,"major":254,"minor":2,"state":["active","sync"],"device":"/dev/vda2","raid_device":1}],"array_size_num":101376,"used_dev_size_num":101376,"name_val":"sysrescue:126","homehost":"sysrescue","uuid_val":"e054553a:fbccdcfe:0ae80bc8:9379377a","events_num":20,"state_list":["clean","resyncing"],"resync_status_percent":27,"creation_time_epoch":1659932149,"update_time_epoch":1659932752} diff --git a/tests/fixtures/generic/mdadm-examine-raid1-spare.json b/tests/fixtures/generic/mdadm-examine-raid1-spare.json index 4a90521ef..27dcf956b 100644 --- a/tests/fixtures/generic/mdadm-examine-raid1-spare.json +++ b/tests/fixtures/generic/mdadm-examine-raid1-spare.json @@ -1 +1 @@ -{"device":"/dev/vda4","magic":"a92b4efc","version":"1.2","feature_map":"0x0","array_uuid":"e054553a:fbccdcfe:0ae80bc8:9379377a","name":"sysrescue:126 (local to host sysrescue)","creation_time":"Sun Aug 7 21:15:49 2022","raid_level":"raid1","raid_devices":2,"avail_dev_size":"202752 sectors (99.00 MiB 103.81 MB)","array_size":"101376 KiB (99.00 MiB 103.81 MB)","data_offset":2048,"super_offset":8,"unused_space":"before=1968 sectors, after=0 sectors","state":"clean","device_uuid":"46b6f457:a3cf3db3:41217976:74a7243c","update_time":"Sun Aug 7 21:39:58 2022","bad_block_log":"512 entries available at offset 16 sectors","checksum":"c265a5d4 - correct","events":52,"device_role":"spare","array_state":"AA ('A' == active, '.' == missing, 'R' == replacing)","array_size_num":101376,"avail_dev_size_num":202752,"unused_space_before":1968,"unused_space_after":0,"name_val":"sysrescue:126","checksum_val":"c265a5d4","checksum_state":"correct","state_list":["clean"],"array_state_list":["active","active"],"creation_time_epoch":1659932149,"update_time_epoch":1659933598} +{"device":"/dev/vda4","magic":"a92b4efc","version":"1.2","feature_map":"0x0","array_uuid":"e054553a:fbccdcfe:0ae80bc8:9379377a","name":"sysrescue:126 (local to host sysrescue)","creation_time":"Sun Aug 7 21:15:49 2022","raid_level":"raid1","raid_devices":2,"avail_dev_size":"202752 sectors (99.00 MiB 103.81 MB)","array_size":"101376 KiB (99.00 MiB 103.81 MB)","data_offset":2048,"super_offset":8,"unused_space":"before=1968 sectors, after=0 sectors","state":"clean","device_uuid":"46b6f457:a3cf3db3:41217976:74a7243c","update_time":"Sun Aug 7 21:39:58 2022","bad_block_log":"512 entries available at offset 16 sectors","checksum":"c265a5d4 - correct","events":"52","device_role":"spare","array_state":"AA ('A' == active, '.' == missing, 'R' == replacing)","array_size_num":101376,"avail_dev_size_num":202752,"unused_space_before":1968,"unused_space_after":0,"name_val":"sysrescue:126","homehost":"sysrescue","events_num":52,"checksum_val":"c265a5d4","checksum_state":"correct","state_list":["clean"],"array_state_list":["active","active"],"creation_time_epoch":1659932149,"update_time_epoch":1659933598} diff --git a/tests/fixtures/generic/mdadm-examine-raid1-syncing.json b/tests/fixtures/generic/mdadm-examine-raid1-syncing.json index 5bd81ed41..7ea029fa8 100644 --- a/tests/fixtures/generic/mdadm-examine-raid1-syncing.json +++ b/tests/fixtures/generic/mdadm-examine-raid1-syncing.json @@ -1 +1 @@ -{"device":"/dev/vda1","magic":"a92b4efc","version":"1.2","feature_map":"0x0","array_uuid":"e054553a:fbccdcfe:0ae80bc8:9379377a","name":"sysrescue:126 (local to host sysrescue)","creation_time":"Sun Aug 7 21:15:49 2022","raid_level":"raid1","raid_devices":2,"avail_dev_size":"202752 sectors (99.00 MiB 103.81 MB)","array_size":"101376 KiB (99.00 MiB 103.81 MB)","data_offset":2048,"super_offset":8,"unused_space":"before=1968 sectors, after=0 sectors","state":"active","device_uuid":"1ec6a01d:e96d7f47:59970cf2:f0c056d2","update_time":"Sun Aug 7 21:15:49 2022","bad_block_log":"512 entries available at offset 16 sectors","checksum":"2e1bdda4 - correct","events":0,"device_role":"Active device 0","array_state":"AA ('A' == active, '.' == missing, 'R' == replacing)","array_size_num":101376,"avail_dev_size_num":202752,"unused_space_before":1968,"unused_space_after":0,"name_val":"sysrescue:126","checksum_val":"2e1bdda4","checksum_state":"correct","state_list":["active"],"array_state_list":["active","active"],"creation_time_epoch":1659932149,"update_time_epoch":1659932149} +{"device":"/dev/vda1","magic":"a92b4efc","version":"1.2","feature_map":"0x0","array_uuid":"e054553a:fbccdcfe:0ae80bc8:9379377a","name":"sysrescue:126 (local to host sysrescue)","creation_time":"Sun Aug 7 21:15:49 2022","raid_level":"raid1","raid_devices":2,"avail_dev_size":"202752 sectors (99.00 MiB 103.81 MB)","array_size":"101376 KiB (99.00 MiB 103.81 MB)","data_offset":2048,"super_offset":8,"unused_space":"before=1968 sectors, after=0 sectors","state":"active","device_uuid":"1ec6a01d:e96d7f47:59970cf2:f0c056d2","update_time":"Sun Aug 7 21:15:49 2022","bad_block_log":"512 entries available at offset 16 sectors","checksum":"2e1bdda4 - correct","events":"0","device_role":"Active device 0","array_state":"AA ('A' == active, '.' == missing, 'R' == replacing)","array_size_num":101376,"avail_dev_size_num":202752,"unused_space_before":1968,"unused_space_after":0,"name_val":"sysrescue:126","homehost":"sysrescue","events_num":0,"checksum_val":"2e1bdda4","checksum_state":"correct","state_list":["active"],"array_state_list":["active","active"],"creation_time_epoch":1659932149,"update_time_epoch":1659932149} diff --git a/tests/fixtures/generic/mdadm-examine.json b/tests/fixtures/generic/mdadm-examine.json index 520a1b1ad..5fa5433b6 100644 --- a/tests/fixtures/generic/mdadm-examine.json +++ b/tests/fixtures/generic/mdadm-examine.json @@ -1 +1 @@ -{"device":"/dev/sdb1","magic":"a92b4efc","version":"1.1","feature_map":"0x1","array_uuid":"85c5b164:d58a5ada:14f5fe07:d642e843","name":"virttest:0","creation_time":"Tue Apr 13 23:22:16 2010","raid_level":"raid1","raid_devices":2,"avail_dev_size":"11721041656 sectors (5.46 TiB 6.00 TB)","array_size":"5860520828 KiB (5.46 TiB 6.00 TB)","data_offset":264,"super_offset":0,"unused_space":"before=80 sectors, after=0 sectors","state":"clean","device_uuid":"813162e5:2e865efe:02ba5570:7003165c","internal_bitmap":"8 sectors from superblock","update_time":"Tue Jul 26 20:16:31 2022","bad_block_log":"512 entries available at offset 72 sectors","checksum":"f141a577 - correct","events":2193679,"device_role":"Active device 0","array_state":"AA ('A' == active, '.' == missing, 'R' == replacing)","array_size_num":5860520828,"avail_dev_size_num":11721041656,"unused_space_before":80,"unused_space_after":0,"name_val":"virttest:0","checksum_val":"f141a577","checksum_state":"correct","state_list":["clean"],"array_state_list":["active","active"],"creation_time_epoch":1271226136,"update_time_epoch":1658891791} +{"device":"/dev/sdb1","magic":"a92b4efc","version":"1.1","feature_map":"0x1","array_uuid":"85c5b164:d58a5ada:14f5fe07:d642e843","name":"virttest:0","creation_time":"Tue Apr 13 23:22:16 2010","raid_level":"raid1","raid_devices":2,"avail_dev_size":"11721041656 sectors (5.46 TiB 6.00 TB)","array_size":"5860520828 KiB (5.46 TiB 6.00 TB)","data_offset":264,"super_offset":0,"unused_space":"before=80 sectors, after=0 sectors","state":"clean","device_uuid":"813162e5:2e865efe:02ba5570:7003165c","internal_bitmap":"8 sectors from superblock","update_time":"Tue Jul 26 20:16:31 2022","bad_block_log":"512 entries available at offset 72 sectors","checksum":"f141a577 - correct","events":"2193679","device_role":"Active device 0","array_state":"AA ('A' == active, '.' == missing, 'R' == replacing)","array_size_num":5860520828,"avail_dev_size_num":11721041656,"unused_space_before":80,"unused_space_after":0,"name_val":"virttest:0","events_num":2193679,"checksum_val":"f141a577","checksum_state":"correct","state_list":["clean"],"array_state_list":["active","active"],"creation_time_epoch":1271226136,"update_time_epoch":1658891791} diff --git a/tests/fixtures/generic/mdadm-query-detail.json b/tests/fixtures/generic/mdadm-query-detail.json index 8277054d2..07e067039 100644 --- a/tests/fixtures/generic/mdadm-query-detail.json +++ b/tests/fixtures/generic/mdadm-query-detail.json @@ -1 +1 @@ -{"device":"/dev/md0","version":"1.1","creation_time":"Tue Apr 13 23:22:16 2010","raid_level":"raid1","array_size":"5860520828 (5.46 TiB 6.00 TB)","used_dev_size":"5860520828 (5.46 TiB 6.00 TB)","raid_devices":2,"total_devices":2,"persistence":"Superblock is persistent","intent_bitmap":"Internal","update_time":"Tue Jul 26 20:16:31 2022","state":"clean","active_devices":2,"working_devices":2,"failed_devices":0,"spare_devices":0,"consistency_policy":"bitmap","name":"virttest:0","uuid":"85c5b164:d58a5ada:14f5fe07:d642e843","events":2193679,"device_table":[{"number":3,"major":8,"minor":17,"state":["active","sync"],"device":"/dev/sdb1","raid_device":0},{"number":2,"major":8,"minor":33,"state":["active","sync"],"device":"/dev/sdc1","raid_device":1}],"array_size_num":5860520828,"used_dev_size_num":5860520828,"name_val":"virttest:0","uuid_val":"85c5b164:d58a5ada:14f5fe07:d642e843","state_list":["clean"],"creation_time_epoch":1271226136,"update_time_epoch":1658891791} +{"device":"/dev/md0","version":"1.1","creation_time":"Tue Apr 13 23:22:16 2010","raid_level":"raid1","array_size":"5860520828 (5.46 TiB 6.00 TB)","used_dev_size":"5860520828 (5.46 TiB 6.00 TB)","raid_devices":2,"total_devices":2,"persistence":"Superblock is persistent","intent_bitmap":"Internal","update_time":"Tue Jul 26 20:16:31 2022","state":"clean","active_devices":2,"working_devices":2,"failed_devices":0,"spare_devices":0,"consistency_policy":"bitmap","name":"virttest:0","uuid":"85c5b164:d58a5ada:14f5fe07:d642e843","events":"2193679","device_table":[{"number":3,"major":8,"minor":17,"state":["active","sync"],"device":"/dev/sdb1","raid_device":0},{"number":2,"major":8,"minor":33,"state":["active","sync"],"device":"/dev/sdc1","raid_device":1}],"array_size_num":5860520828,"used_dev_size_num":5860520828,"name_val":"virttest:0","uuid_val":"85c5b164:d58a5ada:14f5fe07:d642e843","events_num":2193679,"state_list":["clean"],"creation_time_epoch":1271226136,"update_time_epoch":1658891791} diff --git a/tests/fixtures/generic/mdadm-query-raid0-ok.json b/tests/fixtures/generic/mdadm-query-raid0-ok.json index d792f119d..e9da9ec0b 100644 --- a/tests/fixtures/generic/mdadm-query-raid0-ok.json +++ b/tests/fixtures/generic/mdadm-query-raid0-ok.json @@ -1 +1 @@ -{"device":"/dev/md0","version":"1.2","creation_time":"Sun Aug 7 20:56:52 2022","raid_level":"raid0","array_size":"200704 (196.00 MiB 205.52 MB)","raid_devices":2,"total_devices":2,"persistence":"Superblock is persistent","update_time":"Sun Aug 7 20:56:52 2022","state":"clean","active_devices":2,"working_devices":2,"failed_devices":0,"spare_devices":0,"layout":"-unknown-","chunk_size":512,"consistency_policy":"none","name":"sysrescue:0 (local to host sysrescue)","uuid":"7e81a856:abb9c1c2:4b71237a:9778cc66","events":0,"device_table":[{"number":0,"major":254,"minor":1,"state":["active","sync"],"device":"/dev/vda1","raid_device":0},{"number":1,"major":254,"minor":2,"state":["active","sync"],"device":"/dev/vda2","raid_device":1}],"array_size_num":200704,"name_val":"sysrescue:0","uuid_val":"7e81a856:abb9c1c2:4b71237a:9778cc66","state_list":["clean"],"creation_time_epoch":1659931012,"update_time_epoch":1659931012} +{"device":"/dev/md0","version":"1.2","creation_time":"Sun Aug 7 20:56:52 2022","raid_level":"raid0","array_size":"200704 (196.00 MiB 205.52 MB)","raid_devices":2,"total_devices":2,"persistence":"Superblock is persistent","update_time":"Sun Aug 7 20:56:52 2022","state":"clean","active_devices":2,"working_devices":2,"failed_devices":0,"spare_devices":0,"layout":"-unknown-","chunk_size":"512K","consistency_policy":"none","name":"sysrescue:0 (local to host sysrescue)","uuid":"7e81a856:abb9c1c2:4b71237a:9778cc66","events":"0","device_table":[{"number":0,"major":254,"minor":1,"state":["active","sync"],"device":"/dev/vda1","raid_device":0},{"number":1,"major":254,"minor":2,"state":["active","sync"],"device":"/dev/vda2","raid_device":1}],"array_size_num":200704,"name_val":"sysrescue:0","homehost":"sysrescue","uuid_val":"7e81a856:abb9c1c2:4b71237a:9778cc66","chunk_size_num":512,"events_num":0,"state_list":["clean"],"creation_time_epoch":1659931012,"update_time_epoch":1659931012} diff --git a/tests/fixtures/generic/mdadm-query-raid1-failed-and-flags.json b/tests/fixtures/generic/mdadm-query-raid1-failed-and-flags.json index 3afab5d3d..151b8a729 100644 --- a/tests/fixtures/generic/mdadm-query-raid1-failed-and-flags.json +++ b/tests/fixtures/generic/mdadm-query-raid1-failed-and-flags.json @@ -1 +1 @@ -{"device":"/dev/md127","version":"1.2","creation_time":"Sun Aug 7 21:15:49 2022","raid_level":"raid1","array_size":"101376 (99.00 MiB 103.81 MB)","used_dev_size":"101376 (99.00 MiB 103.81 MB)","raid_devices":2,"total_devices":3,"persistence":"Superblock is persistent","update_time":"Sun Aug 7 21:48:58 2022","state":"clean","active_devices":2,"working_devices":2,"failed_devices":1,"spare_devices":0,"consistency_policy":"resync","name":"sysrescue:126 (local to host sysrescue)","uuid":"e054553a:fbccdcfe:0ae80bc8:9379377a","events":78,"device_table":[{"number":0,"major":254,"minor":1,"state":["active","sync"],"device":"/dev/vda1","raid_device":0},{"number":2,"major":254,"minor":2,"state":["active","sync","writemostly","failfast"],"device":"/dev/vda2","raid_device":1},{"number":3,"major":254,"minor":4,"state":["faulty"],"device":"/dev/vda4","raid_device":null}],"array_size_num":101376,"used_dev_size_num":101376,"name_val":"sysrescue:126","uuid_val":"e054553a:fbccdcfe:0ae80bc8:9379377a","state_list":["clean"],"creation_time_epoch":1659932149,"update_time_epoch":1659934138} +{"device":"/dev/md127","version":"1.2","creation_time":"Sun Aug 7 21:15:49 2022","raid_level":"raid1","array_size":"101376 (99.00 MiB 103.81 MB)","used_dev_size":"101376 (99.00 MiB 103.81 MB)","raid_devices":2,"total_devices":3,"persistence":"Superblock is persistent","update_time":"Sun Aug 7 21:48:58 2022","state":"clean","active_devices":2,"working_devices":2,"failed_devices":1,"spare_devices":0,"consistency_policy":"resync","name":"sysrescue:126 (local to host sysrescue)","uuid":"e054553a:fbccdcfe:0ae80bc8:9379377a","events":"78","device_table":[{"number":0,"major":254,"minor":1,"state":["active","sync"],"device":"/dev/vda1","raid_device":0},{"number":2,"major":254,"minor":2,"state":["active","sync","writemostly","failfast"],"device":"/dev/vda2","raid_device":1},{"number":3,"major":254,"minor":4,"state":["faulty"],"device":"/dev/vda4","raid_device":null}],"array_size_num":101376,"used_dev_size_num":101376,"name_val":"sysrescue:126","homehost":"sysrescue","uuid_val":"e054553a:fbccdcfe:0ae80bc8:9379377a","events_num":78,"state_list":["clean"],"creation_time_epoch":1659932149,"update_time_epoch":1659934138} diff --git a/tests/fixtures/generic/mdadm-query-raid1-faulty-and-removed.json b/tests/fixtures/generic/mdadm-query-raid1-faulty-and-removed.json index ce6cedd42..6e366755f 100644 --- a/tests/fixtures/generic/mdadm-query-raid1-faulty-and-removed.json +++ b/tests/fixtures/generic/mdadm-query-raid1-faulty-and-removed.json @@ -1 +1 @@ -{"device":"/dev/md127","version":"1.2","creation_time":"Sun Aug 7 21:15:49 2022","raid_level":"raid1","array_size":"101376 (99.00 MiB 103.81 MB)","used_dev_size":"101376 (99.00 MiB 103.81 MB)","raid_devices":2,"total_devices":1,"persistence":"Superblock is persistent","update_time":"Sun Aug 7 21:32:27 2022","state":"clean, degraded","active_devices":1,"working_devices":1,"failed_devices":0,"spare_devices":0,"consistency_policy":"resync","name":"sysrescue:126 (local to host sysrescue)","uuid":"e054553a:fbccdcfe:0ae80bc8:9379377a","events":31,"device_table":[{"number":0,"major":254,"minor":1,"state":["active","sync"],"device":"/dev/vda1","raid_device":0},{"number":null,"major":0,"minor":0,"state":["removed"],"device":null,"raid_device":1}],"array_size_num":101376,"used_dev_size_num":101376,"name_val":"sysrescue:126","uuid_val":"e054553a:fbccdcfe:0ae80bc8:9379377a","state_list":["clean","degraded"],"creation_time_epoch":1659932149,"update_time_epoch":1659933147} +{"device":"/dev/md127","version":"1.2","creation_time":"Sun Aug 7 21:15:49 2022","raid_level":"raid1","array_size":"101376 (99.00 MiB 103.81 MB)","used_dev_size":"101376 (99.00 MiB 103.81 MB)","raid_devices":2,"total_devices":1,"persistence":"Superblock is persistent","update_time":"Sun Aug 7 21:32:27 2022","state":"clean, degraded","active_devices":1,"working_devices":1,"failed_devices":0,"spare_devices":0,"consistency_policy":"resync","name":"sysrescue:126 (local to host sysrescue)","uuid":"e054553a:fbccdcfe:0ae80bc8:9379377a","events":"31","device_table":[{"number":0,"major":254,"minor":1,"state":["active","sync"],"device":"/dev/vda1","raid_device":0},{"number":null,"major":0,"minor":0,"state":["removed"],"device":null,"raid_device":1}],"array_size_num":101376,"used_dev_size_num":101376,"name_val":"sysrescue:126","homehost":"sysrescue","uuid_val":"e054553a:fbccdcfe:0ae80bc8:9379377a","events_num":31,"state_list":["clean","degraded"],"creation_time_epoch":1659932149,"update_time_epoch":1659933147} diff --git a/tests/fixtures/generic/mdadm-query-raid1-faulty-with-spare.json b/tests/fixtures/generic/mdadm-query-raid1-faulty-with-spare.json index b25b224c1..8bdd6a713 100644 --- a/tests/fixtures/generic/mdadm-query-raid1-faulty-with-spare.json +++ b/tests/fixtures/generic/mdadm-query-raid1-faulty-with-spare.json @@ -1 +1 @@ -{"device":"/dev/md127","version":"1.2","creation_time":"Sun Aug 7 21:15:49 2022","raid_level":"raid1","array_size":"101376 (99.00 MiB 103.81 MB)","used_dev_size":"101376 (99.00 MiB 103.81 MB)","raid_devices":2,"total_devices":3,"persistence":"Superblock is persistent","update_time":"Sun Aug 7 21:43:18 2022","state":"clean, degraded, recovering","active_devices":1,"working_devices":2,"failed_devices":1,"spare_devices":1,"consistency_policy":"resync","rebuild_status":"24% complete","name":"sysrescue:126 (local to host sysrescue)","uuid":"e054553a:fbccdcfe:0ae80bc8:9379377a","events":57,"device_table":[{"number":0,"major":254,"minor":1,"state":["active","sync"],"device":"/dev/vda1","raid_device":0},{"number":3,"major":254,"minor":4,"state":["spare","rebuilding"],"device":"/dev/vda4","raid_device":1},{"number":2,"major":254,"minor":2,"state":["faulty","failfast"],"device":"/dev/vda2","raid_device":null}],"array_size_num":101376,"used_dev_size_num":101376,"name_val":"sysrescue:126","uuid_val":"e054553a:fbccdcfe:0ae80bc8:9379377a","state_list":["clean","degraded","recovering"],"rebuild_status_percent":24,"creation_time_epoch":1659932149,"update_time_epoch":1659933798} +{"device":"/dev/md127","version":"1.2","creation_time":"Sun Aug 7 21:15:49 2022","raid_level":"raid1","array_size":"101376 (99.00 MiB 103.81 MB)","used_dev_size":"101376 (99.00 MiB 103.81 MB)","raid_devices":2,"total_devices":3,"persistence":"Superblock is persistent","update_time":"Sun Aug 7 21:43:18 2022","state":"clean, degraded, recovering","active_devices":1,"working_devices":2,"failed_devices":1,"spare_devices":1,"consistency_policy":"resync","rebuild_status":"24% complete","name":"sysrescue:126 (local to host sysrescue)","uuid":"e054553a:fbccdcfe:0ae80bc8:9379377a","events":"57","device_table":[{"number":0,"major":254,"minor":1,"state":["active","sync"],"device":"/dev/vda1","raid_device":0},{"number":3,"major":254,"minor":4,"state":["spare","rebuilding"],"device":"/dev/vda4","raid_device":1},{"number":2,"major":254,"minor":2,"state":["faulty","failfast"],"device":"/dev/vda2","raid_device":null}],"array_size_num":101376,"used_dev_size_num":101376,"name_val":"sysrescue:126","homehost":"sysrescue","uuid_val":"e054553a:fbccdcfe:0ae80bc8:9379377a","events_num":57,"state_list":["clean","degraded","recovering"],"rebuild_status_percent":24,"creation_time_epoch":1659932149,"update_time_epoch":1659933798} diff --git a/tests/fixtures/generic/mdadm-query-raid1-faulty.json b/tests/fixtures/generic/mdadm-query-raid1-faulty.json index 3a2c8e0f2..5ef2bf5f4 100644 --- a/tests/fixtures/generic/mdadm-query-raid1-faulty.json +++ b/tests/fixtures/generic/mdadm-query-raid1-faulty.json @@ -1 +1 @@ -{"device":"/dev/md126","version":"1.2","creation_time":"Sun Aug 7 21:15:49 2022","raid_level":"raid1","array_size":"101376 (99.00 MiB 103.81 MB)","used_dev_size":"101376 (99.00 MiB 103.81 MB)","raid_devices":2,"total_devices":2,"persistence":"Superblock is persistent","update_time":"Sun Aug 7 21:28:34 2022","state":"clean, degraded","active_devices":1,"working_devices":1,"failed_devices":1,"spare_devices":0,"consistency_policy":"resync","name":"sysrescue:126 (local to host sysrescue)","uuid":"e054553a:fbccdcfe:0ae80bc8:9379377a","events":25,"device_table":[{"number":0,"major":254,"minor":1,"state":["active","sync"],"device":"/dev/vda1","raid_device":0},{"number":null,"major":0,"minor":0,"state":["removed"],"device":null,"raid_device":1},{"number":1,"major":254,"minor":2,"state":["faulty"],"device":"/dev/vda2","raid_device":null}],"array_size_num":101376,"used_dev_size_num":101376,"name_val":"sysrescue:126","uuid_val":"e054553a:fbccdcfe:0ae80bc8:9379377a","state_list":["clean","degraded"],"creation_time_epoch":1659932149,"update_time_epoch":1659932914} +{"device":"/dev/md126","version":"1.2","creation_time":"Sun Aug 7 21:15:49 2022","raid_level":"raid1","array_size":"101376 (99.00 MiB 103.81 MB)","used_dev_size":"101376 (99.00 MiB 103.81 MB)","raid_devices":2,"total_devices":2,"persistence":"Superblock is persistent","update_time":"Sun Aug 7 21:28:34 2022","state":"clean, degraded","active_devices":1,"working_devices":1,"failed_devices":1,"spare_devices":0,"consistency_policy":"resync","name":"sysrescue:126 (local to host sysrescue)","uuid":"e054553a:fbccdcfe:0ae80bc8:9379377a","events":"25","device_table":[{"number":0,"major":254,"minor":1,"state":["active","sync"],"device":"/dev/vda1","raid_device":0},{"number":null,"major":0,"minor":0,"state":["removed"],"device":null,"raid_device":1},{"number":1,"major":254,"minor":2,"state":["faulty"],"device":"/dev/vda2","raid_device":null}],"array_size_num":101376,"used_dev_size_num":101376,"name_val":"sysrescue:126","homehost":"sysrescue","uuid_val":"e054553a:fbccdcfe:0ae80bc8:9379377a","events_num":25,"state_list":["clean","degraded"],"creation_time_epoch":1659932149,"update_time_epoch":1659932914} diff --git a/tests/fixtures/generic/mdadm-query-raid1-ok-0-9.json b/tests/fixtures/generic/mdadm-query-raid1-ok-0-9.json index a0e57efcd..e66d171b1 100644 --- a/tests/fixtures/generic/mdadm-query-raid1-ok-0-9.json +++ b/tests/fixtures/generic/mdadm-query-raid1-ok-0-9.json @@ -1 +1 @@ -{"device":"/dev/md0","version":"0.90","creation_time":"Sun Aug 7 21:52:56 2022","raid_level":"raid1","array_size":"102336 (99.94 MiB 104.79 MB)","used_dev_size":"102336 (99.94 MiB 104.79 MB)","raid_devices":2,"total_devices":2,"preferred_minor":0,"persistence":"Superblock is persistent","update_time":"Sun Aug 7 21:54:35 2022","state":"clean","active_devices":2,"working_devices":2,"failed_devices":0,"spare_devices":0,"consistency_policy":"resync","uuid":"3f720601:29af5283:96fc04a8:108a4af7 (local to host sysrescue)","events":0,"device_table":[{"number":0,"major":254,"minor":1,"state":["active","sync"],"device":"/dev/vda1","raid_device":0},{"number":1,"major":254,"minor":2,"state":["active","sync"],"device":"/dev/vda2","raid_device":1}],"array_size_num":102336,"used_dev_size_num":102336,"uuid_val":"3f720601:29af5283:96fc04a8:108a4af7","state_list":["clean"],"creation_time_epoch":1659934376,"update_time_epoch":1659934475} +{"device":"/dev/md0","version":"0.90","creation_time":"Sun Aug 7 21:52:56 2022","raid_level":"raid1","array_size":"102336 (99.94 MiB 104.79 MB)","used_dev_size":"102336 (99.94 MiB 104.79 MB)","raid_devices":2,"total_devices":2,"preferred_minor":0,"persistence":"Superblock is persistent","update_time":"Sun Aug 7 21:54:35 2022","state":"clean","active_devices":2,"working_devices":2,"failed_devices":0,"spare_devices":0,"consistency_policy":"resync","uuid":"3f720601:29af5283:96fc04a8:108a4af7 (local to host sysrescue)","events":"0.18","device_table":[{"number":0,"major":254,"minor":1,"state":["active","sync"],"device":"/dev/vda1","raid_device":0},{"number":1,"major":254,"minor":2,"state":["active","sync"],"device":"/dev/vda2","raid_device":1}],"array_size_num":102336,"used_dev_size_num":102336,"uuid_val":"3f720601:29af5283:96fc04a8:108a4af7","homehost":"sysrescue","events_maj":0,"events_min":18,"state_list":["clean"],"creation_time_epoch":1659934376,"update_time_epoch":1659934475} diff --git a/tests/fixtures/generic/mdadm-query-raid1-ok-failfast.json b/tests/fixtures/generic/mdadm-query-raid1-ok-failfast.json index d741972df..8327cf8cd 100644 --- a/tests/fixtures/generic/mdadm-query-raid1-ok-failfast.json +++ b/tests/fixtures/generic/mdadm-query-raid1-ok-failfast.json @@ -1 +1 @@ -{"device":"/dev/md127","version":"1.2","creation_time":"Sun Aug 7 21:15:49 2022","raid_level":"raid1","array_size":"101376 (99.00 MiB 103.81 MB)","used_dev_size":"101376 (99.00 MiB 103.81 MB)","raid_devices":2,"total_devices":2,"persistence":"Superblock is persistent","update_time":"Sun Aug 7 21:37:34 2022","state":"clean","active_devices":2,"working_devices":2,"failed_devices":0,"spare_devices":0,"consistency_policy":"resync","name":"sysrescue:126 (local to host sysrescue)","uuid":"e054553a:fbccdcfe:0ae80bc8:9379377a","events":51,"device_table":[{"number":0,"major":254,"minor":1,"state":["active","sync"],"device":"/dev/vda1","raid_device":0},{"number":2,"major":254,"minor":2,"state":["active","sync","failfast"],"device":"/dev/vda2","raid_device":1}],"array_size_num":101376,"used_dev_size_num":101376,"name_val":"sysrescue:126","uuid_val":"e054553a:fbccdcfe:0ae80bc8:9379377a","state_list":["clean"],"creation_time_epoch":1659932149,"update_time_epoch":1659933454} +{"device":"/dev/md127","version":"1.2","creation_time":"Sun Aug 7 21:15:49 2022","raid_level":"raid1","array_size":"101376 (99.00 MiB 103.81 MB)","used_dev_size":"101376 (99.00 MiB 103.81 MB)","raid_devices":2,"total_devices":2,"persistence":"Superblock is persistent","update_time":"Sun Aug 7 21:37:34 2022","state":"clean","active_devices":2,"working_devices":2,"failed_devices":0,"spare_devices":0,"consistency_policy":"resync","name":"sysrescue:126 (local to host sysrescue)","uuid":"e054553a:fbccdcfe:0ae80bc8:9379377a","events":"51","device_table":[{"number":0,"major":254,"minor":1,"state":["active","sync"],"device":"/dev/vda1","raid_device":0},{"number":2,"major":254,"minor":2,"state":["active","sync","failfast"],"device":"/dev/vda2","raid_device":1}],"array_size_num":101376,"used_dev_size_num":101376,"name_val":"sysrescue:126","homehost":"sysrescue","uuid_val":"e054553a:fbccdcfe:0ae80bc8:9379377a","events_num":51,"state_list":["clean"],"creation_time_epoch":1659932149,"update_time_epoch":1659933454} diff --git a/tests/fixtures/generic/mdadm-query-raid1-ok-spare.json b/tests/fixtures/generic/mdadm-query-raid1-ok-spare.json index 2aa630880..f6d9df51a 100644 --- a/tests/fixtures/generic/mdadm-query-raid1-ok-spare.json +++ b/tests/fixtures/generic/mdadm-query-raid1-ok-spare.json @@ -1 +1 @@ -{"device":"/dev/md127","version":"1.2","creation_time":"Sun Aug 7 21:15:49 2022","raid_level":"raid1","array_size":"101376 (99.00 MiB 103.81 MB)","used_dev_size":"101376 (99.00 MiB 103.81 MB)","raid_devices":2,"total_devices":3,"persistence":"Superblock is persistent","update_time":"Sun Aug 7 21:39:58 2022","state":"clean","active_devices":2,"working_devices":3,"failed_devices":0,"spare_devices":1,"consistency_policy":"resync","name":"sysrescue:126 (local to host sysrescue)","uuid":"e054553a:fbccdcfe:0ae80bc8:9379377a","events":52,"device_table":[{"number":0,"major":254,"minor":1,"state":["active","sync"],"device":"/dev/vda1","raid_device":0},{"number":2,"major":254,"minor":2,"state":["active","sync","failfast"],"device":"/dev/vda2","raid_device":1},{"number":3,"major":254,"minor":4,"state":["spare"],"device":"/dev/vda4","raid_device":null}],"array_size_num":101376,"used_dev_size_num":101376,"name_val":"sysrescue:126","uuid_val":"e054553a:fbccdcfe:0ae80bc8:9379377a","state_list":["clean"],"creation_time_epoch":1659932149,"update_time_epoch":1659933598} +{"device":"/dev/md127","version":"1.2","creation_time":"Sun Aug 7 21:15:49 2022","raid_level":"raid1","array_size":"101376 (99.00 MiB 103.81 MB)","used_dev_size":"101376 (99.00 MiB 103.81 MB)","raid_devices":2,"total_devices":3,"persistence":"Superblock is persistent","update_time":"Sun Aug 7 21:39:58 2022","state":"clean","active_devices":2,"working_devices":3,"failed_devices":0,"spare_devices":1,"consistency_policy":"resync","name":"sysrescue:126 (local to host sysrescue)","uuid":"e054553a:fbccdcfe:0ae80bc8:9379377a","events":"52","device_table":[{"number":0,"major":254,"minor":1,"state":["active","sync"],"device":"/dev/vda1","raid_device":0},{"number":2,"major":254,"minor":2,"state":["active","sync","failfast"],"device":"/dev/vda2","raid_device":1},{"number":3,"major":254,"minor":4,"state":["spare"],"device":"/dev/vda4","raid_device":null}],"array_size_num":101376,"used_dev_size_num":101376,"name_val":"sysrescue:126","homehost":"sysrescue","uuid_val":"e054553a:fbccdcfe:0ae80bc8:9379377a","events_num":52,"state_list":["clean"],"creation_time_epoch":1659932149,"update_time_epoch":1659933598} diff --git a/tests/fixtures/generic/mdadm-query-raid1-ok.json b/tests/fixtures/generic/mdadm-query-raid1-ok.json index 4e0173387..7e7f27500 100644 --- a/tests/fixtures/generic/mdadm-query-raid1-ok.json +++ b/tests/fixtures/generic/mdadm-query-raid1-ok.json @@ -1 +1 @@ -{"device":"/dev/md126","version":"1.2","creation_time":"Sun Aug 7 21:15:49 2022","raid_level":"raid1","array_size":"101376 (99.00 MiB 103.81 MB)","used_dev_size":"101376 (99.00 MiB 103.81 MB)","raid_devices":2,"total_devices":2,"persistence":"Superblock is persistent","update_time":"Sun Aug 7 21:20:08 2022","state":"clean","active_devices":2,"working_devices":2,"failed_devices":0,"spare_devices":0,"consistency_policy":"resync","name":"sysrescue:126 (local to host sysrescue)","uuid":"e054553a:fbccdcfe:0ae80bc8:9379377a","events":17,"device_table":[{"number":0,"major":254,"minor":1,"state":["active","sync"],"device":"/dev/vda1","raid_device":0},{"number":1,"major":254,"minor":2,"state":["active","sync"],"device":"/dev/vda2","raid_device":1}],"array_size_num":101376,"used_dev_size_num":101376,"name_val":"sysrescue:126","uuid_val":"e054553a:fbccdcfe:0ae80bc8:9379377a","state_list":["clean"],"creation_time_epoch":1659932149,"update_time_epoch":1659932408} +{"device":"/dev/md126","version":"1.2","creation_time":"Sun Aug 7 21:15:49 2022","raid_level":"raid1","array_size":"101376 (99.00 MiB 103.81 MB)","used_dev_size":"101376 (99.00 MiB 103.81 MB)","raid_devices":2,"total_devices":2,"persistence":"Superblock is persistent","update_time":"Sun Aug 7 21:20:08 2022","state":"clean","active_devices":2,"working_devices":2,"failed_devices":0,"spare_devices":0,"consistency_policy":"resync","name":"sysrescue:126 (local to host sysrescue)","uuid":"e054553a:fbccdcfe:0ae80bc8:9379377a","events":"17","device_table":[{"number":0,"major":254,"minor":1,"state":["active","sync"],"device":"/dev/vda1","raid_device":0},{"number":1,"major":254,"minor":2,"state":["active","sync"],"device":"/dev/vda2","raid_device":1}],"array_size_num":101376,"used_dev_size_num":101376,"name_val":"sysrescue:126","homehost":"sysrescue","uuid_val":"e054553a:fbccdcfe:0ae80bc8:9379377a","events_num":17,"state_list":["clean"],"creation_time_epoch":1659932149,"update_time_epoch":1659932408} diff --git a/tests/fixtures/generic/mdadm-query-raid1-rebuild-failfast.json b/tests/fixtures/generic/mdadm-query-raid1-rebuild-failfast.json index 1dff84061..52e25194e 100644 --- a/tests/fixtures/generic/mdadm-query-raid1-rebuild-failfast.json +++ b/tests/fixtures/generic/mdadm-query-raid1-rebuild-failfast.json @@ -1 +1 @@ -{"device":"/dev/md127","version":"1.2","creation_time":"Sun Aug 7 21:15:49 2022","raid_level":"raid1","array_size":"101376 (99.00 MiB 103.81 MB)","used_dev_size":"101376 (99.00 MiB 103.81 MB)","raid_devices":2,"total_devices":2,"persistence":"Superblock is persistent","update_time":"Sun Aug 7 21:36:17 2022","state":"clean, degraded, recovering","active_devices":1,"working_devices":2,"failed_devices":0,"spare_devices":1,"consistency_policy":"resync","rebuild_status":"20% complete","name":"sysrescue:126 (local to host sysrescue)","uuid":"e054553a:fbccdcfe:0ae80bc8:9379377a","events":37,"device_table":[{"number":0,"major":254,"minor":1,"state":["active","sync"],"device":"/dev/vda1","raid_device":0},{"number":2,"major":254,"minor":2,"state":["failfast","spare","rebuilding"],"device":"/dev/vda2","raid_device":1}],"array_size_num":101376,"used_dev_size_num":101376,"name_val":"sysrescue:126","uuid_val":"e054553a:fbccdcfe:0ae80bc8:9379377a","state_list":["clean","degraded","recovering"],"rebuild_status_percent":20,"creation_time_epoch":1659932149,"update_time_epoch":1659933377} +{"device":"/dev/md127","version":"1.2","creation_time":"Sun Aug 7 21:15:49 2022","raid_level":"raid1","array_size":"101376 (99.00 MiB 103.81 MB)","used_dev_size":"101376 (99.00 MiB 103.81 MB)","raid_devices":2,"total_devices":2,"persistence":"Superblock is persistent","update_time":"Sun Aug 7 21:36:17 2022","state":"clean, degraded, recovering","active_devices":1,"working_devices":2,"failed_devices":0,"spare_devices":1,"consistency_policy":"resync","rebuild_status":"20% complete","name":"sysrescue:126 (local to host sysrescue)","uuid":"e054553a:fbccdcfe:0ae80bc8:9379377a","events":"37","device_table":[{"number":0,"major":254,"minor":1,"state":["active","sync"],"device":"/dev/vda1","raid_device":0},{"number":2,"major":254,"minor":2,"state":["failfast","spare","rebuilding"],"device":"/dev/vda2","raid_device":1}],"array_size_num":101376,"used_dev_size_num":101376,"name_val":"sysrescue:126","homehost":"sysrescue","uuid_val":"e054553a:fbccdcfe:0ae80bc8:9379377a","events_num":37,"state_list":["clean","degraded","recovering"],"rebuild_status_percent":20,"creation_time_epoch":1659932149,"update_time_epoch":1659933377} diff --git a/tests/fixtures/generic/mdadm-query-raid1-spare-writem-rebuild.json b/tests/fixtures/generic/mdadm-query-raid1-spare-writem-rebuild.json index 2549d918f..0c9663c3e 100644 --- a/tests/fixtures/generic/mdadm-query-raid1-spare-writem-rebuild.json +++ b/tests/fixtures/generic/mdadm-query-raid1-spare-writem-rebuild.json @@ -1 +1 @@ -{"device":"/dev/md127","version":"1.2","creation_time":"Sun Aug 7 21:15:49 2022","raid_level":"raid1","array_size":"101376 (99.00 MiB 103.81 MB)","used_dev_size":"101376 (99.00 MiB 103.81 MB)","raid_devices":2,"total_devices":3,"persistence":"Superblock is persistent","update_time":"Sun Aug 7 21:47:42 2022","state":"clean, recovering","active_devices":2,"working_devices":3,"failed_devices":0,"spare_devices":1,"consistency_policy":"resync","rebuild_status":"30% complete","name":"sysrescue:126 (local to host sysrescue)","uuid":"e054553a:fbccdcfe:0ae80bc8:9379377a","events":74,"device_table":[{"number":0,"major":254,"minor":1,"state":["active","sync"],"device":"/dev/vda1","raid_device":0},{"number":2,"major":254,"minor":2,"state":["writemostly","failfast","spare","rebuilding"],"device":"/dev/vda2","raid_device":1},{"number":3,"major":254,"minor":4,"state":["active","sync"],"device":"/dev/vda4","raid_device":1}],"array_size_num":101376,"used_dev_size_num":101376,"name_val":"sysrescue:126","uuid_val":"e054553a:fbccdcfe:0ae80bc8:9379377a","state_list":["clean","recovering"],"rebuild_status_percent":30,"creation_time_epoch":1659932149,"update_time_epoch":1659934062} +{"device":"/dev/md127","version":"1.2","creation_time":"Sun Aug 7 21:15:49 2022","raid_level":"raid1","array_size":"101376 (99.00 MiB 103.81 MB)","used_dev_size":"101376 (99.00 MiB 103.81 MB)","raid_devices":2,"total_devices":3,"persistence":"Superblock is persistent","update_time":"Sun Aug 7 21:47:42 2022","state":"clean, recovering","active_devices":2,"working_devices":3,"failed_devices":0,"spare_devices":1,"consistency_policy":"resync","rebuild_status":"30% complete","name":"sysrescue:126 (local to host sysrescue)","uuid":"e054553a:fbccdcfe:0ae80bc8:9379377a","events":"74","device_table":[{"number":0,"major":254,"minor":1,"state":["active","sync"],"device":"/dev/vda1","raid_device":0},{"number":2,"major":254,"minor":2,"state":["writemostly","failfast","spare","rebuilding"],"device":"/dev/vda2","raid_device":1},{"number":3,"major":254,"minor":4,"state":["active","sync"],"device":"/dev/vda4","raid_device":1}],"array_size_num":101376,"used_dev_size_num":101376,"name_val":"sysrescue:126","homehost":"sysrescue","uuid_val":"e054553a:fbccdcfe:0ae80bc8:9379377a","events_num":74,"state_list":["clean","recovering"],"rebuild_status_percent":30,"creation_time_epoch":1659932149,"update_time_epoch":1659934062} diff --git a/tests/fixtures/generic/mdadm-query-raid1-syncing.json b/tests/fixtures/generic/mdadm-query-raid1-syncing.json index 7a6fb94e3..b847bc02e 100644 --- a/tests/fixtures/generic/mdadm-query-raid1-syncing.json +++ b/tests/fixtures/generic/mdadm-query-raid1-syncing.json @@ -1 +1 @@ -{"device":"/dev/md126","version":"1.2","creation_time":"Sun Aug 7 21:15:49 2022","raid_level":"raid1","array_size":"101376 (99.00 MiB 103.81 MB)","used_dev_size":"101376 (99.00 MiB 103.81 MB)","raid_devices":2,"total_devices":2,"persistence":"Superblock is persistent","update_time":"Sun Aug 7 21:15:49 2022","state":"clean, resyncing","active_devices":2,"working_devices":2,"failed_devices":0,"spare_devices":0,"consistency_policy":"resync","resync_status":"4% complete","name":"sysrescue:126 (local to host sysrescue)","uuid":"e054553a:fbccdcfe:0ae80bc8:9379377a","events":0,"device_table":[{"number":0,"major":254,"minor":1,"state":["active","sync"],"device":"/dev/vda1","raid_device":0},{"number":1,"major":254,"minor":2,"state":["active","sync"],"device":"/dev/vda2","raid_device":1}],"array_size_num":101376,"used_dev_size_num":101376,"name_val":"sysrescue:126","uuid_val":"e054553a:fbccdcfe:0ae80bc8:9379377a","state_list":["clean","resyncing"],"resync_status_percent":4,"creation_time_epoch":1659932149,"update_time_epoch":1659932149} +{"device":"/dev/md126","version":"1.2","creation_time":"Sun Aug 7 21:15:49 2022","raid_level":"raid1","array_size":"101376 (99.00 MiB 103.81 MB)","used_dev_size":"101376 (99.00 MiB 103.81 MB)","raid_devices":2,"total_devices":2,"persistence":"Superblock is persistent","update_time":"Sun Aug 7 21:15:49 2022","state":"clean, resyncing","active_devices":2,"working_devices":2,"failed_devices":0,"spare_devices":0,"consistency_policy":"resync","resync_status":"4% complete","name":"sysrescue:126 (local to host sysrescue)","uuid":"e054553a:fbccdcfe:0ae80bc8:9379377a","events":"0","device_table":[{"number":0,"major":254,"minor":1,"state":["active","sync"],"device":"/dev/vda1","raid_device":0},{"number":1,"major":254,"minor":2,"state":["active","sync"],"device":"/dev/vda2","raid_device":1}],"array_size_num":101376,"used_dev_size_num":101376,"name_val":"sysrescue:126","homehost":"sysrescue","uuid_val":"e054553a:fbccdcfe:0ae80bc8:9379377a","events_num":0,"state_list":["clean","resyncing"],"resync_status_percent":4,"creation_time_epoch":1659932149,"update_time_epoch":1659932149} From 9a50a0cc2ca5979f7c418d3694d88d6d16d117d9 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Tue, 16 Aug 2022 14:43:05 -0700 Subject: [PATCH 080/116] doc update --- docs/parsers/mdadm.md | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/docs/parsers/mdadm.md b/docs/parsers/mdadm.md index f0cd7d221..e12c48215 100644 --- a/docs/parsers/mdadm.md +++ b/docs/parsers/mdadm.md @@ -40,6 +40,15 @@ Schema: "name_val": string, "uuid": string, "uuid_val": string, + "homehost": string, + "container": string, + "container_dev": string, + "container_member": integer, + "controller_guid": string, + "container_guid": string, + "seq": string, + "redundant_hdr": string, + "virtual_disks": integer, "creation_time": string, "creation_time_epoch": integer, # naive timestamp "raid_level": string, @@ -70,13 +79,21 @@ Schema: "checksum": string, "checksum_val": string, "checksum_state": string, - "events": integer, - "chunk_size": integer, + "events": string, + "events_num": integer, + "events_maj": integer, + "events_min": integer, + "chunk_size": string, + "chunk_size_num": integer, "device_role": string, "array_state": string, "array_state_list": [ string ], + "member_arrays": string, + "member_arrays_list": [ + string + ], "consistency_policy": string, "rebuild_status": string, "rebuild_status_percent": integer, @@ -91,6 +108,7 @@ Schema: "working_devices": integer, "failed_devices": integer, "spare_devices": integer, + "physical_disks": integer, "device_table": [ { "number": integer/null, @@ -105,6 +123,8 @@ Schema: ] } + Any fields unspecified above will be string type. + Examples: $ mdadm --query --detail /dev/md0 | jc --mdadm -p From 2e9b9ab987992a96cf28a78b7e3866fa764532ef Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Tue, 16 Aug 2022 15:10:00 -0700 Subject: [PATCH 081/116] tighten up priority parsing --- jc/parsers/syslog_bsd.py | 2 +- tests/fixtures/generic/syslog-3164.json | 2 +- tests/fixtures/generic/syslog-3164.out | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/jc/parsers/syslog_bsd.py b/jc/parsers/syslog_bsd.py index 331fdb571..034687380 100644 --- a/jc/parsers/syslog_bsd.py +++ b/jc/parsers/syslog_bsd.py @@ -117,7 +117,7 @@ def parse( # inspired by https://gist.github.com/miticojo/b16bb13e78572c2d2fac82d9516d5c32 syslog = re.compile(r''' - (?P<\d*>)? + (?P<\d{1,3}>)? (?P[A-Z][a-z][a-z]\s{1,2}\d{1,2}\s\d{2}?:\d{2}:\d{2})?\s (?P[\w][\w\d\.:@-]*)?\s (?P diff --git a/tests/fixtures/generic/syslog-3164.json b/tests/fixtures/generic/syslog-3164.json index 4e8030114..093a62ee7 100644 --- a/tests/fixtures/generic/syslog-3164.json +++ b/tests/fixtures/generic/syslog-3164.json @@ -1 +1 @@ -[{"priority":34,"date":"Oct 11 22:14:15","hostname":"mymachine","tag":"su","content":"'su root' failed for lonvick on /dev/pts/8"},{"priority":null,"date":"Oct 11 22:14:15","hostname":"mymachine","tag":"su","content":"'su root' failed for lonvick on /dev/pts/8"},{"priority":35,"date":"Oct 12 22:14:15","hostname":"client_machine","tag":"su","content":"'su root' failed for joe on /dev/pts/2"},{"priority":35,"date":"Mar 7 04:02:16","hostname":"avas","tag":"clamd","content":"[11165]: /var/amavis/amavis-20040307T033734-10329/parts/part-00003: Worm.Mydoom.F FOUND"},{"priority":null,"date":"Mar 7 04:05:55","hostname":"avas","tag":"clamd","content":"[11240]: /var/amavis/amavis-20040307T035901-10615/parts/part-00002: Worm.SomeFool.Gen-1 FOUND"},{"priority":5,"date":"Mar 7 09:00:51","hostname":"avas","tag":"clamd","content":"[27173]: SelfCheck: Database status OK."},{"priority":null,"date":"Mar 7 05:59:02","hostname":"avas","tag":"clamd","content":"[27173]: Database correctly reloaded (20400 viruses)"},{"priority":null,"date":"Mar 7 04:02:16","hostname":"avas","tag":"clamd","content":"[11165]: /var/amavis/amavis-20040307T033734-10329/parts/part-00003: Worm.Mydoom.F FOUND"},{"priority":null,"date":"Mar 7 04:05:55","hostname":"avas","tag":"clamd","content":"[11240]: /var/amavis/amavis-20040307T035901-10615/parts/part-00002: Worm.SomeFool.Gen-1 FOUND"},{"priority":null,"date":"Mar 7 09:00:51","hostname":"avas","tag":"clamd","content":"[27173]: SelfCheck: Database status OK."},{"priority":null,"date":"Mar 7 05:59:02","hostname":"avas","tag":"clamd","content":"[27173]: Database correctly reloaded (20400 viruses)"},{"priority":null,"date":"Mar 7 11:14:35","hostname":"avas","tag":"dccd","content":"[13284]: 21 requests/sec are too many from anonymous 205.201.1.56,2246"},{"priority":null,"date":"Mar 8 00:22:57","hostname":"avas","tag":"dccifd","content":"[9933]: write(MTA socket,4): Broken pipe"},{"priority":null,"date":"Mar 7 21:23:22","hostname":"avas","tag":"dccifd","content":"[6191]: missing message body"},{"priority":null,"date":"Mar 9 16:05:17","hostname":"avas","tag":"named","content":"[12045]: zone PLNet/IN: refresh: non-authoritative answer from master 10.0.0.253#53"},{"priority":null,"date":"Mar 10 00:38:16","hostname":"avas","tag":"dccifd","content":"[23069]: continue not asking DCC 17 seconds after failure"},{"priority":null,"date":"Mar 10 09:42:11","hostname":"avas","tag":"named","content":"client 127.0.0.1#55524: query: 23.68.27.142.sa-trusted.bondedsender.org IN TXT"},{"priority":null,"date":"Mar 9 03:48:07","hostname":"avas","tag":"dccd","content":"[145]: automatic dbclean; starting `dbclean -DPq -i 1189 -L info,local5.notice -L error,local5.err`"},{"priority":null,"date":"Mar 9 11:58:18","hostname":"avas","tag":"kernel","content":"i810_audio: Connection 0 with codec id 2"},{"priority":null,"date":"Mar 9 19:41:13","hostname":"avas","tag":"dccd","content":"[3004]: \"packet length 44 too small for REPORT\" sent to client 1 at 194.63.250.215,47577"},{"priority":null,"date":"Mar 8 09:01:07","hostname":"avas","tag":"sshd","content":"(pam_unix)[21839]: session opened for user tom by (uid=35567)"},{"priority":null,"date":"Mar 8 03:52:04","hostname":"avas","tag":"dccd","content":"[13284]: 1.2.32 database /home/dcc/dcc_db reopened with 997 MByte window"},{"priority":null,"date":"Mar 8 16:05:26","hostname":"avas","tag":"arpwatch","content":"listening on eth0"},{"priority":null,"date":"Mar 10 10:00:06","hostname":"avas","tag":"named","content":"[6986]: zone PLNet/IN: refresh: non-authoritative answer from master 192.75.26.21#53"},{"priority":null,"date":"Mar 10 10:00:10","hostname":"avas","tag":"named","content":"[6986]: client 127.0.0.1#55867: query: mail.canfor.ca IN MX"},{"priority":null,"date":"Mar 8 15:18:40","hostname":"avas","tag":null,"content":"last message repeated 11 times"},{"priority":null,"date":"Mar 8 15:18:40","hostname":"127:0:ab::1","tag":"sshd","content":"unauthorized request"},{"priority":null,"date":"Mar 8 15:18:40","hostname":"server.example.com","tag":"sshd","content":"unauthorized request"},{"priority":null,"date":"Mar 8 15:18:40","hostname":"192.168.1.1","tag":"sshd","content":"unauthorized request"},{"priority":35,"date":"Mar 8 15:18:40","hostname":"server.example.com","tag":"sshd","content":"unauthorized request"},{"unparsable":"<7>unparsable line"}] +[{"priority":34,"date":"Oct 11 22:14:15","hostname":"mymachine","tag":"su","content":"'su root' failed for lonvick on /dev/pts/8"},{"unparsable":"<3444>Oct 11 22:14:15 mymachine su: 'su root' failed for lonvick on /dev/pts/8"},{"priority":null,"date":"Oct 11 22:14:15","hostname":"mymachine","tag":"su","content":"'su root' failed for lonvick on /dev/pts/8"},{"priority":35,"date":"Oct 12 22:14:15","hostname":"client_machine","tag":"su","content":"'su root' failed for joe on /dev/pts/2"},{"priority":35,"date":"Mar 7 04:02:16","hostname":"avas","tag":"clamd","content":"[11165]: /var/amavis/amavis-20040307T033734-10329/parts/part-00003: Worm.Mydoom.F FOUND"},{"priority":null,"date":"Mar 7 04:05:55","hostname":"avas","tag":"clamd","content":"[11240]: /var/amavis/amavis-20040307T035901-10615/parts/part-00002: Worm.SomeFool.Gen-1 FOUND"},{"priority":5,"date":"Mar 7 09:00:51","hostname":"avas","tag":"clamd","content":"[27173]: SelfCheck: Database status OK."},{"priority":null,"date":"Mar 7 05:59:02","hostname":"avas","tag":"clamd","content":"[27173]: Database correctly reloaded (20400 viruses)"},{"priority":null,"date":"Mar 7 04:02:16","hostname":"avas","tag":"clamd","content":"[11165]: /var/amavis/amavis-20040307T033734-10329/parts/part-00003: Worm.Mydoom.F FOUND"},{"priority":null,"date":"Mar 7 04:05:55","hostname":"avas","tag":"clamd","content":"[11240]: /var/amavis/amavis-20040307T035901-10615/parts/part-00002: Worm.SomeFool.Gen-1 FOUND"},{"priority":null,"date":"Mar 7 09:00:51","hostname":"avas","tag":"clamd","content":"[27173]: SelfCheck: Database status OK."},{"priority":null,"date":"Mar 7 05:59:02","hostname":"avas","tag":"clamd","content":"[27173]: Database correctly reloaded (20400 viruses)"},{"priority":null,"date":"Mar 7 11:14:35","hostname":"avas","tag":"dccd","content":"[13284]: 21 requests/sec are too many from anonymous 205.201.1.56,2246"},{"priority":null,"date":"Mar 8 00:22:57","hostname":"avas","tag":"dccifd","content":"[9933]: write(MTA socket,4): Broken pipe"},{"priority":null,"date":"Mar 7 21:23:22","hostname":"avas","tag":"dccifd","content":"[6191]: missing message body"},{"priority":null,"date":"Mar 9 16:05:17","hostname":"avas","tag":"named","content":"[12045]: zone PLNet/IN: refresh: non-authoritative answer from master 10.0.0.253#53"},{"priority":null,"date":"Mar 10 00:38:16","hostname":"avas","tag":"dccifd","content":"[23069]: continue not asking DCC 17 seconds after failure"},{"priority":null,"date":"Mar 10 09:42:11","hostname":"avas","tag":"named","content":"client 127.0.0.1#55524: query: 23.68.27.142.sa-trusted.bondedsender.org IN TXT"},{"priority":null,"date":"Mar 9 03:48:07","hostname":"avas","tag":"dccd","content":"[145]: automatic dbclean; starting `dbclean -DPq -i 1189 -L info,local5.notice -L error,local5.err`"},{"priority":null,"date":"Mar 9 11:58:18","hostname":"avas","tag":"kernel","content":"i810_audio: Connection 0 with codec id 2"},{"priority":null,"date":"Mar 9 19:41:13","hostname":"avas","tag":"dccd","content":"[3004]: \"packet length 44 too small for REPORT\" sent to client 1 at 194.63.250.215,47577"},{"priority":null,"date":"Mar 8 09:01:07","hostname":"avas","tag":"sshd","content":"(pam_unix)[21839]: session opened for user tom by (uid=35567)"},{"priority":null,"date":"Mar 8 03:52:04","hostname":"avas","tag":"dccd","content":"[13284]: 1.2.32 database /home/dcc/dcc_db reopened with 997 MByte window"},{"priority":null,"date":"Mar 8 16:05:26","hostname":"avas","tag":"arpwatch","content":"listening on eth0"},{"priority":null,"date":"Mar 10 10:00:06","hostname":"avas","tag":"named","content":"[6986]: zone PLNet/IN: refresh: non-authoritative answer from master 192.75.26.21#53"},{"priority":null,"date":"Mar 10 10:00:10","hostname":"avas","tag":"named","content":"[6986]: client 127.0.0.1#55867: query: mail.canfor.ca IN MX"},{"priority":null,"date":"Mar 8 15:18:40","hostname":"avas","tag":null,"content":"last message repeated 11 times"},{"priority":null,"date":"Mar 8 15:18:40","hostname":"127:0:ab::1","tag":"sshd","content":"unauthorized request"},{"priority":null,"date":"Mar 8 15:18:40","hostname":"server.example.com","tag":"sshd","content":"unauthorized request"},{"priority":null,"date":"Mar 8 15:18:40","hostname":"192.168.1.1","tag":"sshd","content":"unauthorized request"},{"priority":35,"date":"Mar 8 15:18:40","hostname":"server.example.com","tag":"sshd","content":"unauthorized request"},{"unparsable":"<7>unparsable line"}] diff --git a/tests/fixtures/generic/syslog-3164.out b/tests/fixtures/generic/syslog-3164.out index d14de4870..bc1170a2e 100644 --- a/tests/fixtures/generic/syslog-3164.out +++ b/tests/fixtures/generic/syslog-3164.out @@ -1,4 +1,5 @@ <34>Oct 11 22:14:15 mymachine su: 'su root' failed for lonvick on /dev/pts/8 +<3444>Oct 11 22:14:15 mymachine su: 'su root' failed for lonvick on /dev/pts/8 Oct 11 22:14:15 mymachine su: 'su root' failed for lonvick on /dev/pts/8 <35>Oct 12 22:14:15 client_machine su: 'su root' failed for joe on /dev/pts/2 <35>Mar 7 04:02:16 avas clamd[11165]: /var/amavis/amavis-20040307T033734-10329/parts/part-00003: Worm.Mydoom.F FOUND From 4b9ed91546ebca77d9e73b9b01dacac1fdbc16a0 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Tue, 16 Aug 2022 15:59:08 -0700 Subject: [PATCH 082/116] regex formatting --- jc/parsers/syslog_bsd.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/jc/parsers/syslog_bsd.py b/jc/parsers/syslog_bsd.py index 034687380..44677c158 100644 --- a/jc/parsers/syslog_bsd.py +++ b/jc/parsers/syslog_bsd.py @@ -118,8 +118,10 @@ def parse( # inspired by https://gist.github.com/miticojo/b16bb13e78572c2d2fac82d9516d5c32 syslog = re.compile(r''' (?P<\d{1,3}>)? - (?P[A-Z][a-z][a-z]\s{1,2}\d{1,2}\s\d{2}?:\d{2}:\d{2})?\s - (?P[\w][\w\d\.:@-]*)?\s + (?P
+ (?P[A-Z][a-z][a-z]\s{1,2}\d{1,2}\s\d{2}?:\d{2}:\d{2})?\s + (?P[\w][\w\d\.:@-]*)?\s + ) (?P (?P\w+)? (?P.*) From d5839e199fc0a4175eb38aad20017a69c5be5ba7 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Tue, 16 Aug 2022 16:15:03 -0700 Subject: [PATCH 083/116] add syslog-bsd-s streaming parser --- jc/lib.py | 3 +- jc/parsers/syslog_bsd_s.py | 181 +++++++++++++++++++++++++++++++++++++ 2 files changed, 183 insertions(+), 1 deletion(-) create mode 100644 jc/parsers/syslog_bsd_s.py diff --git a/jc/lib.py b/jc/lib.py index e88613cfa..692424ad6 100644 --- a/jc/lib.py +++ b/jc/lib.py @@ -94,8 +94,9 @@ 'stat', 'stat-s', 'sysctl', - 'syslog-bsd', 'syslog', + 'syslog-bsd', + 'syslog-bsd-s', 'systemctl', 'systemctl-lj', 'systemctl-ls', diff --git a/jc/parsers/syslog_bsd_s.py b/jc/parsers/syslog_bsd_s.py new file mode 100644 index 000000000..5536f1947 --- /dev/null +++ b/jc/parsers/syslog_bsd_s.py @@ -0,0 +1,181 @@ +"""jc - JSON Convert Syslog RFC 3164 string streaming parser + +> This streaming parser outputs JSON Lines (cli) or returns an Iterable of +> Dictionaries (module) + +This parser accepts a single syslog line string or multiple syslog lines +separated by newlines. A warning message to `STDERR` will be printed if an +unparsable line is found unless `--quiet` or `quiet=True` is used. + +Usage (cli): + + $ echo '<34>Oct 11 22:14:15 mymachine su: su ro...' | jc --syslog-bsd-s + +Usage (module): + + import jc + + result = jc.parse('syslog_bsd_s', syslog_command_output.splitlines()) + for item in result: + # do something + +Schema: + + { + "priority": integer/null, + "date": string, + "hostname": string, + "tag": string/null, + "content": string, + "unparsable": string, # [0] + + # below object only exists if using -qq or ignore_exceptions=True + "_jc_meta": { + "success": boolean, # false if error parsing + "error": string, # exists if "success" is false + "line": string # exists if "success" is false + } + } + + [0] this field exists if the syslog line is not parsable. The value + is the original syslog line. + +Examples: + + $ cat syslog.txt | jc --syslog-bsd-s -p + {"priority":34,"date":"Oct 11 22:14:15","hostname":"mymachine","t...} + ... + + $ cat syslog.txt | jc --syslog-bsd-s -p -r + {"priority":"34","date":"Oct 11 22:14:15","hostname":"mymachine","...} + ... +""" +from typing import Dict, Iterable, Union +import re +import jc.utils +from jc.streaming import ( + add_jc_meta, streaming_input_type_check, streaming_line_input_type_check, raise_or_yield +) +from jc.exceptions import ParseError + + +class info(): + """Provides parser metadata (version, author, etc.)""" + version = '1.0' + description = 'Syslog RFC 3164 string streaming parser' + author = 'Kelly Brazil' + author_email = 'kellyjonbrazil@gmail.com' + compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd'] + streaming = True + + +__version__ = info.version + + +def _process(proc_data: Dict) -> Dict: + """ + Final processing to conform to the schema. + + Parameters: + + proc_data: (Dictionary) raw structured data to process + + Returns: + + Dictionary. Structured data to conform to the schema. + """ + int_list = {'priority'} + + for key in proc_data: + if key in int_list: + proc_data[key] = jc.utils.convert_to_int(proc_data[key]) + + return proc_data + + +@add_jc_meta +def parse( + data: Iterable[str], + raw: bool = False, + quiet: bool = False, + ignore_exceptions: bool = False +) -> Union[Iterable[Dict], tuple]: + """ + Main text parsing generator function. Returns an iterable object. + + Parameters: + + data: (iterable) line-based text data to parse + (e.g. sys.stdin or str.splitlines()) + + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + ignore_exceptions: (boolean) ignore parsing exceptions if True + + + Returns: + + Iterable of Dictionaries + """ + jc.utils.compatibility(__name__, info.compatible, quiet) + streaming_input_type_check(data) + + # inspired by https://gist.github.com/miticojo/b16bb13e78572c2d2fac82d9516d5c32 + syslog = re.compile(r''' + (?P<\d{1,3}>)? + (?P
+ (?P[A-Z][a-z][a-z]\s{1,2}\d{1,2}\s\d{2}?:\d{2}:\d{2})?\s + (?P[\w][\w\d\.:@-]*)?\s + ) + (?P + (?P\w+)? + (?P.*) + ) + ''', re.VERBOSE + ) + + for line in data: + try: + streaming_line_input_type_check(line) + output_line: Dict = {} + + #skip blank lines + if not line.strip(): + continue + + syslog_match = syslog.match(line) + if syslog_match: + priority = None + if syslog_match.group('priority'): + priority = syslog_match.group('priority')[1:-1] + + # check for missing tag + hostname = syslog_match.group('host') + tag = syslog_match.group('tag') + content = syslog_match.group('content') + if hostname: + if hostname.endswith(':'): + content = tag + content + tag = None + hostname = hostname[:-1] + + output_line = { + 'priority': priority, + 'date': syslog_match.group('date'), + 'hostname': hostname, + # 'raw_msg': syslog_match.group('msg'), + 'tag': tag, + 'content': content.lstrip(' :').rstrip() + } + + else: + output_line = { + 'unparsable': line + } + + + if output_line: + yield output_line if raw else _process(output_line) + + except Exception as e: + yield raise_or_yield(ignore_exceptions, e, line) From 80b4f7d0376b6de8f933b02d9708baa8d65fffbc Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Tue, 16 Aug 2022 16:33:50 -0700 Subject: [PATCH 084/116] add streamin syslog parsers --- README.md | 4 +- completions/jc_bash_completion.sh | 2 +- completions/jc_zsh_completion.sh | 6 +- docs/parsers/syslog.md | 4 +- docs/parsers/syslog_bsd_s.md | 89 ++++++++++ docs/parsers/syslog_s.md | 87 ++++++++++ jc/lib.py | 1 + jc/parsers/syslog.py | 4 +- jc/parsers/syslog_s.py | 270 ++++++++++++++++++++++++++++++ man/jc.1 | 14 +- 10 files changed, 471 insertions(+), 10 deletions(-) create mode 100644 docs/parsers/syslog_bsd_s.md create mode 100644 docs/parsers/syslog_s.md create mode 100644 jc/parsers/syslog_s.py diff --git a/README.md b/README.md index d3bd1f6da..38e652902 100644 --- a/README.md +++ b/README.md @@ -235,8 +235,10 @@ option. | ` --stat` | `stat` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/stat) | | ` --stat-s` | `stat` command streaming parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/stat_s) | | ` --sysctl` | `sysctl` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/sysctl) | -| ` --syslog-bsd` | Syslog RFC 3164 string parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/syslog_bsd) | | ` --syslog` | Syslog RFC 5424 string parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/syslog) | +| ` --syslog-s` | Syslog RFC 5424 string streaming parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/syslog_s) | +| ` --syslog-bsd` | Syslog RFC 3164 string parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/syslog_bsd) | +| ` --syslog-bsd-s` | Syslog RFC 3164 string streaming parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/syslog_bsd_s) | | ` --systemctl` | `systemctl` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/systemctl) | | ` --systemctl-lj` | `systemctl list-jobs` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/systemctl_lj) | | ` --systemctl-ls` | `systemctl list-sockets` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/systemctl_ls) | diff --git a/completions/jc_bash_completion.sh b/completions/jc_bash_completion.sh index 1e9ba714a..4431b3f85 100644 --- a/completions/jc_bash_completion.sh +++ b/completions/jc_bash_completion.sh @@ -4,7 +4,7 @@ _jc() jc_about_options jc_about_mod_options jc_help_options jc_special_options jc_commands=(acpi airport arp blkid chage cksum crontab date df dig dmidecode dpkg du env file finger free git gpg hciconfig id ifconfig iostat iptables iw jobs last lastb ls lsblk lsmod lsof lsusb md5 md5sum mdadm mount mpstat netstat nmcli ntpq pidstat ping ping6 pip pip3 postconf printenv ps route rpm rsync sfdisk sha1sum sha224sum sha256sum sha384sum sha512sum shasum ss stat sum sysctl systemctl systeminfo timedatectl top tracepath tracepath6 traceroute traceroute6 ufw uname update-alternatives upower uptime vdir vmstat w wc who xrandr zipinfo) - jc_parsers=(--acpi --airport --airport-s --arp --asciitable --asciitable-m --blkid --cef --chage --cksum --crontab --crontab-u --csv --csv-s --date --df --dig --dir --dmidecode --dpkg-l --du --email-address --env --file --finger --free --fstab --git-log --git-log-s --gpg --group --gshadow --hash --hashsum --hciconfig --history --hosts --id --ifconfig --ini --iostat --iostat-s --ip-address --iptables --iso-datetime --iw-scan --jar-manifest --jobs --jwt --kv --last --ls --ls-s --lsblk --lsmod --lsof --lsusb --m3u --mdadm --mount --mpstat --mpstat-s --netstat --nmcli --ntpq --passwd --pidstat --pidstat-s --ping --ping-s --pip-list --pip-show --plist --postconf --ps --route --rpm-qi --rsync --rsync-s --sfdisk --shadow --ss --stat --stat-s --sysctl --syslog-bsd --syslog --systemctl --systemctl-lj --systemctl-ls --systemctl-luf --systeminfo --time --timedatectl --timestamp --top --top-s --tracepath --traceroute --ufw --ufw-appinfo --uname --update-alt-gs --update-alt-q --upower --uptime --url --vmstat --vmstat-s --w --wc --who --x509-cert --xml --xrandr --yaml --zipinfo) + jc_parsers=(--acpi --airport --airport-s --arp --asciitable --asciitable-m --blkid --cef --chage --cksum --crontab --crontab-u --csv --csv-s --date --df --dig --dir --dmidecode --dpkg-l --du --email-address --env --file --finger --free --fstab --git-log --git-log-s --gpg --group --gshadow --hash --hashsum --hciconfig --history --hosts --id --ifconfig --ini --iostat --iostat-s --ip-address --iptables --iso-datetime --iw-scan --jar-manifest --jobs --jwt --kv --last --ls --ls-s --lsblk --lsmod --lsof --lsusb --m3u --mdadm --mount --mpstat --mpstat-s --netstat --nmcli --ntpq --passwd --pidstat --pidstat-s --ping --ping-s --pip-list --pip-show --plist --postconf --ps --route --rpm-qi --rsync --rsync-s --sfdisk --shadow --ss --stat --stat-s --sysctl --syslog --syslog-s --syslog-bsd --syslog-bsd-s --systemctl --systemctl-lj --systemctl-ls --systemctl-luf --systeminfo --time --timedatectl --timestamp --top --top-s --tracepath --traceroute --ufw --ufw-appinfo --uname --update-alt-gs --update-alt-q --upower --uptime --url --vmstat --vmstat-s --w --wc --who --x509-cert --xml --xrandr --yaml --zipinfo) jc_options=(--force-color -C --debug -d --monochrome -m --pretty -p --quiet -q --raw -r --time-out -t --unbuffer -u --yaml-out -y) jc_about_options=(--about -a) jc_about_mod_options=(--pretty -p --yaml-out -y --monochrome -m --force-color -C) diff --git a/completions/jc_zsh_completion.sh b/completions/jc_zsh_completion.sh index c527cd31f..206e5afc3 100644 --- a/completions/jc_zsh_completion.sh +++ b/completions/jc_zsh_completion.sh @@ -95,7 +95,7 @@ _jc() { 'xrandr:run "xrandr" command with magic syntax.' 'zipinfo:run "zipinfo" command with magic syntax.' ) - jc_parsers=(--acpi --airport --airport-s --arp --asciitable --asciitable-m --blkid --cef --chage --cksum --crontab --crontab-u --csv --csv-s --date --df --dig --dir --dmidecode --dpkg-l --du --email-address --env --file --finger --free --fstab --git-log --git-log-s --gpg --group --gshadow --hash --hashsum --hciconfig --history --hosts --id --ifconfig --ini --iostat --iostat-s --ip-address --iptables --iso-datetime --iw-scan --jar-manifest --jobs --jwt --kv --last --ls --ls-s --lsblk --lsmod --lsof --lsusb --m3u --mdadm --mount --mpstat --mpstat-s --netstat --nmcli --ntpq --passwd --pidstat --pidstat-s --ping --ping-s --pip-list --pip-show --plist --postconf --ps --route --rpm-qi --rsync --rsync-s --sfdisk --shadow --ss --stat --stat-s --sysctl --syslog-bsd --syslog --systemctl --systemctl-lj --systemctl-ls --systemctl-luf --systeminfo --time --timedatectl --timestamp --top --top-s --tracepath --traceroute --ufw --ufw-appinfo --uname --update-alt-gs --update-alt-q --upower --uptime --url --vmstat --vmstat-s --w --wc --who --x509-cert --xml --xrandr --yaml --zipinfo) + jc_parsers=(--acpi --airport --airport-s --arp --asciitable --asciitable-m --blkid --cef --chage --cksum --crontab --crontab-u --csv --csv-s --date --df --dig --dir --dmidecode --dpkg-l --du --email-address --env --file --finger --free --fstab --git-log --git-log-s --gpg --group --gshadow --hash --hashsum --hciconfig --history --hosts --id --ifconfig --ini --iostat --iostat-s --ip-address --iptables --iso-datetime --iw-scan --jar-manifest --jobs --jwt --kv --last --ls --ls-s --lsblk --lsmod --lsof --lsusb --m3u --mdadm --mount --mpstat --mpstat-s --netstat --nmcli --ntpq --passwd --pidstat --pidstat-s --ping --ping-s --pip-list --pip-show --plist --postconf --ps --route --rpm-qi --rsync --rsync-s --sfdisk --shadow --ss --stat --stat-s --sysctl --syslog --syslog-s --syslog-bsd --syslog-bsd-s --systemctl --systemctl-lj --systemctl-ls --systemctl-luf --systeminfo --time --timedatectl --timestamp --top --top-s --tracepath --traceroute --ufw --ufw-appinfo --uname --update-alt-gs --update-alt-q --upower --uptime --url --vmstat --vmstat-s --w --wc --who --x509-cert --xml --xrandr --yaml --zipinfo) jc_parsers_describe=( '--acpi:`acpi` command parser' '--airport:`airport -I` command parser' @@ -182,8 +182,10 @@ _jc() { '--stat:`stat` command parser' '--stat-s:`stat` command streaming parser' '--sysctl:`sysctl` command parser' - '--syslog-bsd:Syslog RFC 3164 string parser' '--syslog:Syslog RFC 5424 string parser' + '--syslog-s:Syslog RFC 5424 string streaming parser' + '--syslog-bsd:Syslog RFC 3164 string parser' + '--syslog-bsd-s:Syslog RFC 3164 string streaming parser' '--systemctl:`systemctl` command parser' '--systemctl-lj:`systemctl list-jobs` command parser' '--systemctl-ls:`systemctl list-sockets` command parser' diff --git a/docs/parsers/syslog.md b/docs/parsers/syslog.md index ca0eba981..a6981d199 100644 --- a/docs/parsers/syslog.md +++ b/docs/parsers/syslog.md @@ -58,7 +58,7 @@ Blank values converted to `null`/`None` Examples: - $ cat syslog.txt| jc --syslog -p + $ cat syslog.txt | jc --syslog -p [ { "priority": 35, @@ -90,7 +90,7 @@ Examples: } ] - $ cat syslog.txt| jc --syslog -p -r + $ cat syslog.txt | jc --syslog -p -r [ { "priority": "35", diff --git a/docs/parsers/syslog_bsd_s.md b/docs/parsers/syslog_bsd_s.md new file mode 100644 index 000000000..90217cbd4 --- /dev/null +++ b/docs/parsers/syslog_bsd_s.md @@ -0,0 +1,89 @@ +[Home](https://kellyjonbrazil.github.io/jc/) + + +# jc.parsers.syslog\_bsd\_s + +jc - JSON Convert Syslog RFC 3164 string streaming parser + +> This streaming parser outputs JSON Lines (cli) or returns an Iterable of +> Dictionaries (module) + +This parser accepts a single syslog line string or multiple syslog lines +separated by newlines. A warning message to `STDERR` will be printed if an +unparsable line is found unless `--quiet` or `quiet=True` is used. + +Usage (cli): + + $ echo '<34>Oct 11 22:14:15 mymachine su: su ro...' | jc --syslog-bsd-s + +Usage (module): + + import jc + + result = jc.parse('syslog_bsd_s', syslog_command_output.splitlines()) + for item in result: + # do something + +Schema: + + { + "priority": integer/null, + "date": string, + "hostname": string, + "tag": string/null, + "content": string, + "unparsable": string, # [0] + + # below object only exists if using -qq or ignore_exceptions=True + "_jc_meta": { + "success": boolean, # false if error parsing + "error": string, # exists if "success" is false + "line": string # exists if "success" is false + } + } + + [0] this field exists if the syslog line is not parsable. The value + is the original syslog line. + +Examples: + + $ cat syslog.txt | jc --syslog-bsd-s -p + {"priority":34,"date":"Oct 11 22:14:15","hostname":"mymachine","t...} + ... + + $ cat syslog.txt | jc --syslog-bsd-s -p -r + {"priority":"34","date":"Oct 11 22:14:15","hostname":"mymachine","...} + ... + + + +### parse + +```python +@add_jc_meta +def parse(data: Iterable[str], + raw: bool = False, + quiet: bool = False, + ignore_exceptions: bool = False) -> Union[Iterable[Dict], tuple] +``` + +Main text parsing generator function. Returns an iterable object. + +Parameters: + + data: (iterable) line-based text data to parse + (e.g. sys.stdin or str.splitlines()) + + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + ignore_exceptions: (boolean) ignore parsing exceptions if True + + +Returns: + + Iterable of Dictionaries + +### Parser Information +Compatibility: linux, darwin, cygwin, win32, aix, freebsd + +Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/docs/parsers/syslog_s.md b/docs/parsers/syslog_s.md new file mode 100644 index 000000000..c21e78dee --- /dev/null +++ b/docs/parsers/syslog_s.md @@ -0,0 +1,87 @@ +[Home](https://kellyjonbrazil.github.io/jc/) + + +# jc.parsers.syslog\_s + +jc - JSON Convert Syslog RFC 5424 string streaming parser + +> This streaming parser outputs JSON Lines (cli) or returns an Iterable of +> Dictionaries (module) + +This parser accepts a single syslog line string or multiple syslog lines +separated by newlines. A warning message to `STDERR` will be printed if an +unparsable line is found unless `--quiet` or `quiet=True` is used. + +The `timestamp_epoch` calculated timestamp field is naive. (i.e. based on +the local time of the system the parser is run on) + +The `timestamp_epoch_utc` calculated timestamp field is timezone-aware and +is only available if the timezone field is UTC. + +Usage (cli): + + $ echo <165>1 2003-08-24T05:14:15.000003-07:00 192.0... | jc --syslog-s + +Usage (module): + + import jc + + result = jc.parse('syslog_s', syslog_command_output.splitlines()) + for item in result: + # do something + +Schema: + + { + "foo": string, + + # below object only exists if using -qq or ignore_exceptions=True + "_jc_meta": { + "success": boolean, # false if error parsing + "error": string, # exists if "success" is false + "line": string # exists if "success" is false + } + } + +Examples: + + $ cat syslog.txt | jc --syslog-s -p + {"priority":165,"version":1,"timestamp":"2003-08-24T05:14:15.000003-...} + ... + + $ cat syslog.txt | jc --syslog-s -p -r + {"priority":"165","version":"1","timestamp":"2003-08-24T05:14:15.000...} + ... + + + +### parse + +```python +@add_jc_meta +def parse(data: Iterable[str], + raw: bool = False, + quiet: bool = False, + ignore_exceptions: bool = False) -> Union[Iterable[Dict], tuple] +``` + +Main text parsing generator function. Returns an iterable object. + +Parameters: + + data: (iterable) line-based text data to parse + (e.g. sys.stdin or str.splitlines()) + + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + ignore_exceptions: (boolean) ignore parsing exceptions if True + + +Returns: + + Iterable of Dictionaries + +### Parser Information +Compatibility: linux, darwin, cygwin, win32, aix, freebsd + +Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/jc/lib.py b/jc/lib.py index 692424ad6..92026dbff 100644 --- a/jc/lib.py +++ b/jc/lib.py @@ -95,6 +95,7 @@ 'stat-s', 'sysctl', 'syslog', + 'syslog-s', 'syslog-bsd', 'syslog-bsd-s', 'systemctl', diff --git a/jc/parsers/syslog.py b/jc/parsers/syslog.py index 5915c71bd..ea81aae27 100644 --- a/jc/parsers/syslog.py +++ b/jc/parsers/syslog.py @@ -53,7 +53,7 @@ Examples: - $ cat syslog.txt| jc --syslog -p + $ cat syslog.txt | jc --syslog -p [ { "priority": 35, @@ -85,7 +85,7 @@ } ] - $ cat syslog.txt| jc --syslog -p -r + $ cat syslog.txt | jc --syslog -p -r [ { "priority": "35", diff --git a/jc/parsers/syslog_s.py b/jc/parsers/syslog_s.py new file mode 100644 index 000000000..c6ede9853 --- /dev/null +++ b/jc/parsers/syslog_s.py @@ -0,0 +1,270 @@ +"""jc - JSON Convert Syslog RFC 5424 string streaming parser + +> This streaming parser outputs JSON Lines (cli) or returns an Iterable of +> Dictionaries (module) + +This parser accepts a single syslog line string or multiple syslog lines +separated by newlines. A warning message to `STDERR` will be printed if an +unparsable line is found unless `--quiet` or `quiet=True` is used. + +The `timestamp_epoch` calculated timestamp field is naive. (i.e. based on +the local time of the system the parser is run on) + +The `timestamp_epoch_utc` calculated timestamp field is timezone-aware and +is only available if the timezone field is UTC. + +Usage (cli): + + $ echo <165>1 2003-08-24T05:14:15.000003-07:00 192.0... | jc --syslog-s + +Usage (module): + + import jc + + result = jc.parse('syslog_s', syslog_command_output.splitlines()) + for item in result: + # do something + +Schema: + + { + "foo": string, + + # below object only exists if using -qq or ignore_exceptions=True + "_jc_meta": { + "success": boolean, # false if error parsing + "error": string, # exists if "success" is false + "line": string # exists if "success" is false + } + } + +Examples: + + $ cat syslog.txt | jc --syslog-s -p + {"priority":165,"version":1,"timestamp":"2003-08-24T05:14:15.000003-...} + ... + + $ cat syslog.txt | jc --syslog-s -p -r + {"priority":"165","version":"1","timestamp":"2003-08-24T05:14:15.000...} + ... +""" +from typing import List, Dict, Iterable, Union, Optional +import re +import jc.utils +from jc.streaming import ( + add_jc_meta, streaming_input_type_check, streaming_line_input_type_check, raise_or_yield +) +from jc.exceptions import ParseError + + +class info(): + """Provides parser metadata (version, author, etc.)""" + version = '1.0' + description = 'Syslog RFC 5424 string streaming parser' + author = 'Kelly Brazil' + author_email = 'kellyjonbrazil@gmail.com' + compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd'] + streaming = True + + +__version__ = info.version + + +# fix escape chars specified in syslog RFC 5424 +# https://www.rfc-editor.org/rfc/rfc5424.html#section-6 +escape_map = { + r'\\': '\\', + r'\"': r'"', + r'\]': r']' +} + + +def _extract_structs(structs_string: str) -> List[str]: + struct_match = re.compile(r'(?P\[.+?(? Optional[str]: + ident = re.compile(r'\[(?P[^\[\=\x22\]\x20]{1,32})\s') + ident_match = ident.search(struct_string) + if ident_match: + return ident_match.group('ident') + return None + + +def _extract_kv(struct_string) -> List[Dict]: + key_vals = re.compile(r'(?P\w+)=(?P\"[^\"]*\")') + key_vals_match = key_vals.findall(struct_string) + kv_list = [] + + if key_vals_match: + for kv in key_vals_match: + key, val = kv + + # fixup escaped characters + for esc, esc_sub in escape_map.items(): + val = val.replace(esc, esc_sub) + + kv_list.append({key: val[1:-1]}) + + return kv_list + + +def _process(proc_data: Dict) -> Dict: + """ + Final processing to conform to the schema. + + Parameters: + + proc_data: (Dictionary) raw structured data to process + + Returns: + + Dictionary. Structured data to conform to the schema. + """ + int_list = {'priority', 'version', 'proc_id'} + + for key, value in proc_data.items(): + # remove any spaces around values + if proc_data[key]: + proc_data[key] = value.strip() + + # add timestamp fields + if 'timestamp' in proc_data and proc_data['timestamp']: + format = (1300, 1310) + dt = jc.utils.timestamp(proc_data['timestamp'], format) + proc_data['timestamp_epoch'] = dt.naive + proc_data['timestamp_epoch_utc'] = dt.utc + + # fixup escaped characters + if 'message' in proc_data and proc_data['message']: + for esc, esc_sub in escape_map.items(): + proc_data['message'] = proc_data['message'].replace(esc, esc_sub) + + # parse identity and key value pairs in the structured data section + if 'structured_data' in proc_data and proc_data['structured_data']: + structs_list = [] + structs = _extract_structs(proc_data['structured_data']) + + for a_struct in structs: + struct_obj = { + 'identity': _extract_ident(a_struct) + } + + my_values = {} + + for val_obj in _extract_kv(a_struct): + my_values.update(val_obj) + + struct_obj.update({'parameters': my_values}) # type: ignore + structs_list.append(struct_obj) + + proc_data['structured_data'] = structs_list + + # integer conversions + for key in proc_data: + if key in int_list: + proc_data[key] = jc.utils.convert_to_int(proc_data[key]) + + return proc_data + + +@add_jc_meta +def parse( + data: Iterable[str], + raw: bool = False, + quiet: bool = False, + ignore_exceptions: bool = False +) -> Union[Iterable[Dict], tuple]: + """ + Main text parsing generator function. Returns an iterable object. + + Parameters: + + data: (iterable) line-based text data to parse + (e.g. sys.stdin or str.splitlines()) + + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + ignore_exceptions: (boolean) ignore parsing exceptions if True + + + Returns: + + Iterable of Dictionaries + """ + jc.utils.compatibility(__name__, info.compatible, quiet) + streaming_input_type_check(data) + + # inspired by https://regex101.com/library/Wgbxn2 + syslog = re.compile(r''' + (?P<(\d|\d{2}|1[1-8]\d|19[01])>)? + (?P\d{1,2})?\s* + (?P-| + (?P[12]\d{3})- + (?P0\d|[1][012])- + (?P[012]\d|3[01])T + (?P[01]\d|2[0-4]): + (?P[0-5]\d): + (?P[0-5]\d|60)(?#60seconds can be used for leap year!)(?:\. + (?P\d{1,6}))? + (?PZ|[+-]\d{2}:\d{2})(?#=timezone))\s + (?P[\S]{1,255})\s + (?P[\S]{1,48})\s + (?P[\S]{1,128})\s + (?P[\S]{1,32})\s + (?P-|(?:\[.+?(?.+))? + ''', re.VERBOSE + ) + + for line in data: + try: + streaming_line_input_type_check(line) + output_line: Dict = {} + + #skip blank lines + if not line.strip(): + continue + + syslog_match = syslog.match(line) + if syslog_match: + syslog_dict = syslog_match.groupdict() + for item in syslog_dict: + if syslog_dict[item] == '-': + syslog_dict[item] = None + + priority = None + + if syslog_dict['priority']: + priority = syslog_dict['priority'][1:-1] + + output_line = { + 'priority': priority, + 'version': syslog_dict['version'], + 'timestamp': syslog_dict['timestamp'], + 'hostname': syslog_dict['hostname'], + 'appname': syslog_dict['appname'], + 'proc_id': syslog_dict['procid'], + 'msg_id': syslog_dict['msgid'], + 'structured_data': syslog_dict['structureddata'], + 'message': syslog_dict['msg'] + } + + else: + output_line = { + 'unparsable': line + } + + if output_line: + yield output_line if raw else _process(output_line) + + except Exception as e: + yield raise_or_yield(ignore_exceptions, e, line) diff --git a/man/jc.1 b/man/jc.1 index 4e8dcfadc..85c1f566e 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -442,6 +442,16 @@ PLIST file parser \fB--sysctl\fP `sysctl` command parser +.TP +.B +\fB--syslog\fP +Syslog RFC 5424 string parser + +.TP +.B +\fB--syslog-s\fP +Syslog RFC 5424 string streaming parser + .TP .B \fB--syslog-bsd\fP @@ -449,8 +459,8 @@ Syslog RFC 3164 string parser .TP .B -\fB--syslog\fP -Syslog RFC 5424 string parser +\fB--syslog-bsd-s\fP +Syslog RFC 3164 string streaming parser .TP .B From 8cbd92ccb60af666671abef152f90390e0a9f3b2 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Tue, 16 Aug 2022 16:39:00 -0700 Subject: [PATCH 085/116] add unparsable warning message --- jc/parsers/syslog_bsd_s.py | 4 ++++ jc/parsers/syslog_s.py | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/jc/parsers/syslog_bsd_s.py b/jc/parsers/syslog_bsd_s.py index 5536f1947..0daab32de 100644 --- a/jc/parsers/syslog_bsd_s.py +++ b/jc/parsers/syslog_bsd_s.py @@ -173,6 +173,10 @@ def parse( 'unparsable': line } + if not quiet: + jc.utils.warning_message( + [f'Unparsable line found: {line}'] + ) if output_line: yield output_line if raw else _process(output_line) diff --git a/jc/parsers/syslog_s.py b/jc/parsers/syslog_s.py index c6ede9853..ba0e4dd9a 100644 --- a/jc/parsers/syslog_s.py +++ b/jc/parsers/syslog_s.py @@ -263,6 +263,11 @@ def parse( 'unparsable': line } + if not quiet: + jc.utils.warning_message( + [f'Unparsable line found: {line}'] + ) + if output_line: yield output_line if raw else _process(output_line) From 5f43abc908f816551688bf556b7d4b21219f9043 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Tue, 16 Aug 2022 17:00:09 -0700 Subject: [PATCH 086/116] strip newline at the end of each line string --- jc/parsers/syslog_bsd_s.py | 4 ++-- jc/parsers/syslog_s.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/jc/parsers/syslog_bsd_s.py b/jc/parsers/syslog_bsd_s.py index 0daab32de..c50e2f980 100644 --- a/jc/parsers/syslog_bsd_s.py +++ b/jc/parsers/syslog_bsd_s.py @@ -170,12 +170,12 @@ def parse( else: output_line = { - 'unparsable': line + 'unparsable': line.rstrip() } if not quiet: jc.utils.warning_message( - [f'Unparsable line found: {line}'] + [f'Unparsable line found: {line.rstrip()}'] ) if output_line: diff --git a/jc/parsers/syslog_s.py b/jc/parsers/syslog_s.py index ba0e4dd9a..fdb972ae8 100644 --- a/jc/parsers/syslog_s.py +++ b/jc/parsers/syslog_s.py @@ -260,12 +260,12 @@ def parse( else: output_line = { - 'unparsable': line + 'unparsable': line.rstrip() } if not quiet: jc.utils.warning_message( - [f'Unparsable line found: {line}'] + [f'Unparsable line found: {line.rstrip()}'] ) if output_line: From e777992b3edd6d073e32a2c95040fce36bc4caca Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Tue, 16 Aug 2022 17:12:09 -0700 Subject: [PATCH 087/116] add schema docs --- docs/parsers/syslog_s.md | 25 ++++++++++++++++++++++++- jc/parsers/syslog_s.py | 25 ++++++++++++++++++++++++- 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/docs/parsers/syslog_s.md b/docs/parsers/syslog_s.md index c21e78dee..5cd594f4d 100644 --- a/docs/parsers/syslog_s.md +++ b/docs/parsers/syslog_s.md @@ -33,7 +33,25 @@ Usage (module): Schema: { - "foo": string, + "priority": integer, + "version": integer, + "timestamp": string, + "timestamp_epoch": integer, # [0] + "timestamp_epoch_utc": integer, # [1] + "hostname": string, + "appname": string, + "proc_id": integer, + "msg_id": string, + "structured_data": [ + { + "identity": string, + "parameters": { + "": string + } + } + ], + "message": string, + "unparsable": string # [2] # below object only exists if using -qq or ignore_exceptions=True "_jc_meta": { @@ -43,6 +61,11 @@ Schema: } } + [0] naive timestamp if "timestamp" field is parsable, else null + [1] timezone aware timestamp availabe for UTC, else null + [2] this field exists if the syslog line is not parsable. The value + is the original syslog line. + Examples: $ cat syslog.txt | jc --syslog-s -p diff --git a/jc/parsers/syslog_s.py b/jc/parsers/syslog_s.py index fdb972ae8..62cf8888a 100644 --- a/jc/parsers/syslog_s.py +++ b/jc/parsers/syslog_s.py @@ -28,7 +28,25 @@ Schema: { - "foo": string, + "priority": integer, + "version": integer, + "timestamp": string, + "timestamp_epoch": integer, # [0] + "timestamp_epoch_utc": integer, # [1] + "hostname": string, + "appname": string, + "proc_id": integer, + "msg_id": string, + "structured_data": [ + { + "identity": string, + "parameters": { + "": string + } + } + ], + "message": string, + "unparsable": string # [2] # below object only exists if using -qq or ignore_exceptions=True "_jc_meta": { @@ -38,6 +56,11 @@ } } + [0] naive timestamp if "timestamp" field is parsable, else null + [1] timezone aware timestamp availabe for UTC, else null + [2] this field exists if the syslog line is not parsable. The value + is the original syslog line. + Examples: $ cat syslog.txt | jc --syslog-s -p From e20357663df896757c881471831a43b7f80c36bf Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Tue, 16 Aug 2022 17:13:53 -0700 Subject: [PATCH 088/116] doc formatting --- docs/parsers/syslog_s.md | 34 +++++++++++++++++----------------- jc/parsers/syslog_s.py | 34 +++++++++++++++++----------------- 2 files changed, 34 insertions(+), 34 deletions(-) diff --git a/docs/parsers/syslog_s.md b/docs/parsers/syslog_s.md index 5cd594f4d..f05500540 100644 --- a/docs/parsers/syslog_s.md +++ b/docs/parsers/syslog_s.md @@ -34,24 +34,24 @@ Schema: { "priority": integer, - "version": integer, - "timestamp": string, - "timestamp_epoch": integer, # [0] - "timestamp_epoch_utc": integer, # [1] - "hostname": string, - "appname": string, - "proc_id": integer, - "msg_id": string, - "structured_data": [ - { - "identity": string, - "parameters": { - "": string - } + "version": integer, + "timestamp": string, + "timestamp_epoch": integer, # [0] + "timestamp_epoch_utc": integer, # [1] + "hostname": string, + "appname": string, + "proc_id": integer, + "msg_id": string, + "structured_data": [ + { + "identity": string, + "parameters": { + "": string } - ], - "message": string, - "unparsable": string # [2] + } + ], + "message": string, + "unparsable": string # [2] # below object only exists if using -qq or ignore_exceptions=True "_jc_meta": { diff --git a/jc/parsers/syslog_s.py b/jc/parsers/syslog_s.py index 62cf8888a..317bf039d 100644 --- a/jc/parsers/syslog_s.py +++ b/jc/parsers/syslog_s.py @@ -29,24 +29,24 @@ { "priority": integer, - "version": integer, - "timestamp": string, - "timestamp_epoch": integer, # [0] - "timestamp_epoch_utc": integer, # [1] - "hostname": string, - "appname": string, - "proc_id": integer, - "msg_id": string, - "structured_data": [ - { - "identity": string, - "parameters": { - "": string - } + "version": integer, + "timestamp": string, + "timestamp_epoch": integer, # [0] + "timestamp_epoch_utc": integer, # [1] + "hostname": string, + "appname": string, + "proc_id": integer, + "msg_id": string, + "structured_data": [ + { + "identity": string, + "parameters": { + "": string } - ], - "message": string, - "unparsable": string # [2] + } + ], + "message": string, + "unparsable": string # [2] # below object only exists if using -qq or ignore_exceptions=True "_jc_meta": { From 8682a6bd0a8a2efea79ccba8bba19f5ee2f766eb Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Tue, 16 Aug 2022 21:08:44 -0700 Subject: [PATCH 089/116] normalize key names --- jc/parsers/cef.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/jc/parsers/cef.py b/jc/parsers/cef.py index bee30ebd9..c63306a72 100644 --- a/jc/parsers/cef.py +++ b/jc/parsers/cef.py @@ -197,10 +197,11 @@ def _process(proc_data: List[Dict]) -> List[Dict]: for esc, esc_sub in escape_map.items(): item[key] = item[key].replace(esc, esc_sub) - # remove any quotation marks from key names - if '"' in key: - new_key = key.replace('"', '') - item[new_key] = item.pop(key) + # normalize keynames + new_key = key.strip() + new_key = re.sub(r'[^a-zA-Z0-9]', '_', new_key) + new_key = new_key.strip('_') + item[new_key] = item.pop(key) return proc_data From cadf7e732eec7efea3a11cedd5ef7d41a4bb12e1 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Wed, 17 Aug 2022 12:54:12 -0700 Subject: [PATCH 090/116] doc formatting --- docs/parsers/syslog.md | 3 ++- docs/parsers/syslog_s.md | 2 ++ jc/parsers/syslog.py | 3 ++- jc/parsers/syslog_s.py | 2 ++ man/jc.1 | 2 +- 5 files changed, 9 insertions(+), 3 deletions(-) diff --git a/docs/parsers/syslog.md b/docs/parsers/syslog.md index a6981d199..2d05e5a9d 100644 --- a/docs/parsers/syslog.md +++ b/docs/parsers/syslog.md @@ -25,7 +25,8 @@ Usage (module): result = jc.parse('syslog', syslog_string) Schema: -Blank values converted to `null`/`None` + +Blank values converted to `null`/`None`. [ { diff --git a/docs/parsers/syslog_s.md b/docs/parsers/syslog_s.md index f05500540..f2604845b 100644 --- a/docs/parsers/syslog_s.md +++ b/docs/parsers/syslog_s.md @@ -32,6 +32,8 @@ Usage (module): Schema: +Blank values converted to `null`/`None`. + { "priority": integer, "version": integer, diff --git a/jc/parsers/syslog.py b/jc/parsers/syslog.py index ea81aae27..afaf6f0d5 100644 --- a/jc/parsers/syslog.py +++ b/jc/parsers/syslog.py @@ -20,7 +20,8 @@ result = jc.parse('syslog', syslog_string) Schema: -Blank values converted to `null`/`None` + +Blank values converted to `null`/`None`. [ { diff --git a/jc/parsers/syslog_s.py b/jc/parsers/syslog_s.py index 317bf039d..896af5629 100644 --- a/jc/parsers/syslog_s.py +++ b/jc/parsers/syslog_s.py @@ -27,6 +27,8 @@ Schema: +Blank values converted to `null`/`None`. + { "priority": integer, "version": integer, diff --git a/man/jc.1 b/man/jc.1 index 85c1f566e..13bc13012 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -1,4 +1,4 @@ -.TH jc 1 2022-08-16 1.21.0 "JSON Convert" +.TH jc 1 2022-08-17 1.21.0 "JSON Convert" .SH NAME \fBjc\fP \- JSON Convert JSONifies the output of many CLI tools and file-types .SH SYNOPSIS From 07c65351d555fc2bebe515bb20a91ba58b25d0ad Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Wed, 17 Aug 2022 15:20:18 -0700 Subject: [PATCH 091/116] rename keys to match spec. attempt type conversions for extended fields. --- jc/parsers/cef.py | 106 +++++++++++++++++++++++++++++++++++++++------- 1 file changed, 91 insertions(+), 15 deletions(-) diff --git a/jc/parsers/cef.py b/jc/parsers/cef.py index c63306a72..497f67ca2 100644 --- a/jc/parsers/cef.py +++ b/jc/parsers/cef.py @@ -21,14 +21,28 @@ Schema: +See: https://www.microfocus.com/documentation/arcsight/arcsight-smartconnectors-8.3/cef-implementation-standard/Content/CEF/Chapter%201%20What%20is%20CEF.htm + [ { - "cef": string, - "bar": boolean, - "baz": integer + "deviceVendor": string, + "deviceProduct": string, + "deviceVersion": string, + "deviceEventClassId": string, + "name": string, + "agentSeverity": string/integer, + "agentSeverityString": string, + "agentSeverityNum": integer, + "CEF_Version": integer, + string/integer/float, # [0] + string } ] + [0] Will attempt to convert extended fields to the type specified in the + CEF specification. If conversion fails, then the field will remain + a string. + Examples: $ cef | jc --cef -p @@ -114,13 +128,13 @@ def _pycef_parse(str_input): # Since these values are set by their position in the header, it's # easy to know which is which. - values["DeviceVendor"] = spl[1] - values["DeviceProduct"] = spl[2] - values["DeviceVersion"] = spl[3] - values["DeviceEventClassID"] = spl[4] - values["Name"] = spl[5] + values["deviceVendor"] = spl[1] + values["deviceProduct"] = spl[2] + values["deviceVersion"] = spl[3] + values["deviceEventClassId"] = spl[4] + values["name"] = spl[5] if len(spl) > 6: - values["Severity"] = spl[6] + values["agentSeverity"] = spl[6] # The first value is actually the CEF version, formatted like # "CEF:#". Ignore anything before that (like a date from a syslog message). @@ -130,7 +144,7 @@ def _pycef_parse(str_input): if cef_start == -1: raise ParseError('Invalid CEF string.') (cef, version) = spl[0][cef_start:].split(':') - values["CEFVersion"] = version + values["CEF_Version"] = version # The ugly, gnarly regex here finds a single key=value pair, # taking into account multiple whitespaces, escaped '=' and '|' @@ -141,6 +155,34 @@ def _pycef_parse(str_input): # Split the tuples and put them into the dictionary values[i[0]] = i[1] + # set defined types for extended fields + # see https://www.microfocus.com/documentation/arcsight/arcsight-smartconnectors-8.3/cef-implementation-standard/#CEF/Chapter%202%20ArcSight%20Extension.htm + extended_ints = { + 'spid', 'customerKey', 'deviceTranslatedZoneKey', 'oldFileSize', + 'destination TranslatedPort', 'cn3', 'source TranslatedPort', 'in', 'fsize', 'slat', + 'dpid', 'cnt', 'agentZoneKey', 'out', 'type', 'eventId', 'dlong', 'cn2', + 'deviceDirection', 'spt', 'agentTranslatedZoneKey', 'sTranslatedZoneKey', 'cn1', + 'slong', 'dZoneKey', 'deviceZoneKey', 'dvcpid', 'dpt', 'dTranslatedZoneKey', 'dlat', + 'sZoneKey' + } + + extended_floats = { + 'cfp1', 'cfp2', 'cfp3', 'cfp4' + } + + for k, v in values.items(): + if k in extended_ints: + try: + values[k] = int(v) + except Exception: + pass + + if k in extended_floats: + try: + values[k] = float(v) + except Exception: + pass + # Process custom field labels for key in list(values.keys()): # If the key string ends with Label, replace it in the appropriate @@ -188,14 +230,34 @@ def _process(proc_data: List[Dict]) -> List[Dict]: r'\r': '\r' } + int_list = {'CEF_Version'} + + severity_map = { + None: 'Unknown', + 0: 'Low', + 1: 'Low', + 2: 'Low', + 3: 'Low', + 4: 'Medium', + 5: 'Medium', + 6: 'Medium', + 7: 'High', + 8: 'High', + 9: 'Very-High', + 10: 'Very-High' + } + + severity_set = {'unknown', 'low', 'medium', 'high', 'very-high'} + for item in proc_data: for key, value in item.copy().items(): - # remove any spaces around values - item[key] = value.strip() + if isinstance(item[key], str): + # remove any spaces around values + item[key] = value.strip() - # fixup escaped characters - for esc, esc_sub in escape_map.items(): - item[key] = item[key].replace(esc, esc_sub) + # fixup escaped characters + for esc, esc_sub in escape_map.items(): + item[key] = item[key].replace(esc, esc_sub) # normalize keynames new_key = key.strip() @@ -203,6 +265,20 @@ def _process(proc_data: List[Dict]) -> List[Dict]: new_key = new_key.strip('_') item[new_key] = item.pop(key) + # integer conversions + if key in int_list: + item[key] = jc.utils.convert_to_int(item[key]) + + # set SeverityString and SeverityNum: + if 'agentSeverity' in item: + if isinstance(item['agentSeverity'], str) and item['agentSeverity'].lower() in severity_set: + item['agentSeverityString'] = item['agentSeverity'] + item['agentSeverityNum'] = None + else: + item['agentSeverity'] = int(item['agentSeverity']) + item['agentSeverityString'] = severity_map[item['agentSeverity']] + item['agentSeverityNum'] = item['agentSeverity'] + return proc_data From 42b24d71e92933fc41ecc5c88a3f1152fa54a55d Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Wed, 17 Aug 2022 17:45:47 -0700 Subject: [PATCH 092/116] add CEF time formats --- jc/utils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/jc/utils.py b/jc/utils.py index ebdc5ae01..de2894e3c 100644 --- a/jc/utils.py +++ b/jc/utils.py @@ -375,6 +375,10 @@ def _parse_dt(dt_string, format_hint=None): {'id': 1100, 'format': '%a %b %d %H:%M:%S %Y %z', 'locale': None}, # git date output: Thu Mar 5 09:17:40 2020 -0800 {'id': 1300, 'format': '%Y-%m-%dT%H:%M:%S.%f%Z', 'locale': None}, # ISO Format with UTC (found in syslog 5424): 2003-10-11T22:14:15.003Z {'id': 1310, 'format': '%Y-%m-%dT%H:%M:%S.%f', 'locale': None}, # ISO Format without TZ (found in syslog 5424): 2003-10-11T22:14:15.003 + {'id': 1400, 'format': '%b %d %Y %H:%M:%S.%f UTC', 'locale': None}, # CEF Format with UTC: Nov 08 2022 12:30:00.111 UTC + {'id': 1410, 'format': '%b %d %Y %H:%M:%S.%f', 'locale': None}, # CEF Format without TZ: Nov 08 2022 12:30:00.111 + {'id': 1420, 'format': '%b %d %Y %H:%M:%S UTC', 'locale': None}, # CEF Format with UTC without microseconds: Nov 08 2022 12:30:00 UTC + {'id': 1430, 'format': '%b %d %Y %H:%M:%S', 'locale': None}, # CEF Format without TZ or microseconds: Nov 08 2022 12:30:00 {'id': 1500, 'format': '%Y-%m-%d %H:%M', 'locale': None}, # en_US.UTF-8 local format (found in who cli output): 2021-03-23 00:14 {'id': 1600, 'format': '%m/%d/%Y %I:%M %p', 'locale': None}, # Windows english format (found in dir cli output): 12/07/2019 02:09 AM {'id': 1700, 'format': '%m/%d/%Y, %I:%M:%S %p', 'locale': None}, # Windows english format wint non-UTC tz (found in systeminfo cli output): 3/22/2021, 1:15:51 PM (UTC-0600) From 5b597b6583c8faccf2b2c910e1a5043509ded1ed Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Wed, 17 Aug 2022 17:46:24 -0700 Subject: [PATCH 093/116] move conversions into process(). initial timestamp fields working --- jc/parsers/cef.py | 94 +++++++++++++++++++++++++++-------------------- 1 file changed, 54 insertions(+), 40 deletions(-) diff --git a/jc/parsers/cef.py b/jc/parsers/cef.py index 497f67ca2..999003266 100644 --- a/jc/parsers/cef.py +++ b/jc/parsers/cef.py @@ -23,6 +23,8 @@ See: https://www.microfocus.com/documentation/arcsight/arcsight-smartconnectors-8.3/cef-implementation-standard/Content/CEF/Chapter%201%20What%20is%20CEF.htm +> Note: Special characters in key names will be converted to underscores. + [ { "deviceVendor": string, @@ -155,46 +157,6 @@ def _pycef_parse(str_input): # Split the tuples and put them into the dictionary values[i[0]] = i[1] - # set defined types for extended fields - # see https://www.microfocus.com/documentation/arcsight/arcsight-smartconnectors-8.3/cef-implementation-standard/#CEF/Chapter%202%20ArcSight%20Extension.htm - extended_ints = { - 'spid', 'customerKey', 'deviceTranslatedZoneKey', 'oldFileSize', - 'destination TranslatedPort', 'cn3', 'source TranslatedPort', 'in', 'fsize', 'slat', - 'dpid', 'cnt', 'agentZoneKey', 'out', 'type', 'eventId', 'dlong', 'cn2', - 'deviceDirection', 'spt', 'agentTranslatedZoneKey', 'sTranslatedZoneKey', 'cn1', - 'slong', 'dZoneKey', 'deviceZoneKey', 'dvcpid', 'dpt', 'dTranslatedZoneKey', 'dlat', - 'sZoneKey' - } - - extended_floats = { - 'cfp1', 'cfp2', 'cfp3', 'cfp4' - } - - for k, v in values.items(): - if k in extended_ints: - try: - values[k] = int(v) - except Exception: - pass - - if k in extended_floats: - try: - values[k] = float(v) - except Exception: - pass - - # Process custom field labels - for key in list(values.keys()): - # If the key string ends with Label, replace it in the appropriate - # custom field - if key[-5:] == "Label": - customlabel = key[:-5] - # Find the corresponding customfield and replace with the label - for customfield in list(values.keys()): - if customfield == customlabel: - values[values[key]] = values[customfield] - del values[customfield] - del values[key] else: raise ParseError('Could not parse record. Is it valid CEF format?') @@ -249,7 +211,59 @@ def _process(proc_data: List[Dict]) -> List[Dict]: severity_set = {'unknown', 'low', 'medium', 'high', 'very-high'} + # set defined types for extended fields + # see https://www.microfocus.com/documentation/arcsight/arcsight-smartconnectors-8.3/cef-implementation-standard/#CEF/Chapter%202%20ArcSight%20Extension.htm + extended_ints = { + 'spid', 'customerKey', 'deviceTranslatedZoneKey', 'oldFileSize', + 'destinationTranslatedPort', 'cn3', 'sourceTranslatedPort', 'in', 'fsize', 'slat', + 'dpid', 'cnt', 'agentZoneKey', 'out', 'type', 'eventId', 'dlong', 'cn2', + 'deviceDirection', 'spt', 'agentTranslatedZoneKey', 'sTranslatedZoneKey', 'cn1', + 'slong', 'dZoneKey', 'deviceZoneKey', 'dvcpid', 'dpt', 'dTranslatedZoneKey', 'dlat', + 'sZoneKey' + } + + extended_floats = { + 'cfp1', 'cfp2', 'cfp3', 'cfp4' + } + + extended_dt = { + 'deviceCustomDate1', 'deviceCustomDate2', 'end', 'fileCreateTime', + 'fileModificationTime', 'flexDate1', 'oldFileCreateTime', 'oldFileModificationTime', + 'rt', 'start', 'art' + } + for item in proc_data: + for key, value in item.copy().items(): + if key in extended_ints: + try: + item[key] = int(value) + except Exception: + pass + + if key in extended_floats: + try: + item[key] = float(value) + except Exception: + pass + + if key in extended_dt: + dt = jc.utils.timestamp(item[key]) + item[key + '_epoch'] = dt.naive + item[key + '_epoch_utc'] = dt.utc + + # Process custom field labels (from pycef library) + for key in list(item.keys()): + # If the key string ends with Label, replace it in the appropriate + # custom field + if key[-5:] == "Label": + customlabel = key[:-5] + # Find the corresponding customfield and replace with the label + for customfield in list(item.keys()): + if customfield == customlabel: + item[item[key]] = item[customfield] + del item[customfield] + del item[key] + for key, value in item.copy().items(): if isinstance(item[key], str): # remove any spaces around values From d71a7fbbedfdfd2a8b4a515e5cddef84710dffd3 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 18 Aug 2022 10:55:37 -0700 Subject: [PATCH 094/116] cleanup custom field processing and add datetime support --- jc/parsers/cef.py | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/jc/parsers/cef.py b/jc/parsers/cef.py index 999003266..1b78b6954 100644 --- a/jc/parsers/cef.py +++ b/jc/parsers/cef.py @@ -251,19 +251,33 @@ def _process(proc_data: List[Dict]) -> List[Dict]: item[key + '_epoch'] = dt.naive item[key + '_epoch_utc'] = dt.utc - # Process custom field labels (from pycef library) - for key in list(item.keys()): - # If the key string ends with Label, replace it in the appropriate - # custom field - if key[-5:] == "Label": + # Process custom field labels (adapted from pycef library) + cleanup_list = [] + custom_fields = list(item.keys()) + for key in custom_fields: + if key.endswith('Label'): customlabel = key[:-5] - # Find the corresponding customfield and replace with the label - for customfield in list(item.keys()): + for customfield in custom_fields: + # check for normal custom fields if customfield == customlabel: item[item[key]] = item[customfield] - del item[customfield] - del item[key] + cleanup_list.append(customfield) + cleanup_list.append(key) + # check for datetime objects + if customfield == customlabel + '_epoch': + item[item[key] + '_epoch'] = item[customfield] + cleanup_list.append(customfield) + + if customfield == customlabel + '_epoch_utc': + item[item[key] + '_epoch_utc'] = item[customfield] + cleanup_list.append(customfield) + + # cleanup extra custom fields + for key in cleanup_list: + del item[key] + + # more normalization for key, value in item.copy().items(): if isinstance(item[key], str): # remove any spaces around values From 780eb0555311a04c768b5b7796b5fc97b103cbe4 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 18 Aug 2022 11:19:11 -0700 Subject: [PATCH 095/116] doc update --- docs/parsers/cef.md | 94 +++++++++++++++++++++++++++++++++++++++------ jc/parsers/cef.py | 72 ++++++++++++++++++++++++++++++---- man/jc.1 | 2 +- 3 files changed, 148 insertions(+), 20 deletions(-) diff --git a/docs/parsers/cef.md b/docs/parsers/cef.md index 1cbec06ca..c59bbb2bd 100644 --- a/docs/parsers/cef.md +++ b/docs/parsers/cef.md @@ -5,15 +5,21 @@ jc - JSON Convert CEF string parser -This is a best-effort parser since there are so many variations to CEF -formatting from different vendors. If you require special handling for your -CEF input, you can copy this parser code to the `jc` pluggin directory for -your system and modify it to suit your needs. +This parser conforms to the Microfocus Arcsight CEF specification. If you +require special handling for your CEF input, you can copy this parser code +to the `jc` pluggin directory for your system and modify it to suit your +needs. This parser will accept a single CEF string or multiple CEF string lines. Any text before "CEF" will be ignored. Syslog and CEF escaped characters -(`\\`, `\\"`, `\\]`, `\\|`, `\\n`, `\\r`) are unescaped. To preserve -escaping, use the `--raw` or `raw=True` option in the `parse()` function. +(`\\`, `\\"`, `\\]`, `\\|`, `\\n`, `\\r`) are unescaped. + +Extended fields, as defined in the CEF specification, are relabeled +and the values are converted to their respective types. Extra naive and +UTC epoch timestamps are added where appropriate per the CEF specification. + +To preserve escaping, original keynames, and value types use the `--raw` or +`raw=True` option in the `parse()` function. Usage (cli): @@ -26,21 +32,87 @@ Usage (module): Schema: +See: https://www.microfocus.com/documentation/arcsight/arcsight-smartconnectors-8.3/cef-implementation-standard/Content/CEF/Chapter%201%20What%20is%20CEF.htm + +> Note: Special characters in key names will be converted to underscores. + [ { - "cef": string, - "bar": boolean, - "baz": integer + "deviceVendor": string, + "deviceProduct": string, + "deviceVersion": string, + "deviceEventClassId": string, + "name": string, + "agentSeverity": string/integer, + "agentSeverityString": string, + "agentSeverityNum": integer, + "CEF_Version": integer, + string/integer/float, # [0] + "_epoch": integer, # [1] + "_epoch_utc": integer, # [2] + string } ] + [0] Will attempt to convert extended fields to the type specified in the + CEF specification. If conversion fails, then the field will remain + a string. + [1] Naive calculated epoch timestamp + [2] Timezone-aware calculated epoch timestamp. (UTC only) This value + will be null if a UTC timezone cannot be extracted from the original + timestamp string value. + Examples: $ cef | jc --cef -p - [] + [ + { + "deviceVendor": "Trend Micro", + "deviceProduct": "Deep Security Agent", + "deviceVersion": "", + "deviceEventClassId": "4000000", + "name": "Eicar_test_file", + "agentSeverity": 6, + "CEF_Version": 0, + "dvchost": "hostname", + "string": "hello \"world\"!", + "start": "Nov 08 2020 12:30:00.111 UTC", + "start_epoch": 1604867400, + "start_epoch_utc": 1604838600, + "Host_ID": 1, + "Quarantine": 205, + "myDate": "Nov 08 2022 12:30:00.111", + "myDate_epoch": 1667939400, + "myDate_epoch_utc": null, + "myFloat": 3.14, + "agentSeverityString": "Medium", + "agentSeverityNum": 6 + } + ] $ cef | jc --cef -p -r - [] + [ + { + "deviceVendor": "Trend Micro", + "deviceProduct": "Deep Security Agent", + "deviceVersion": "", + "deviceEventClassId": "4000000", + "name": "Eicar_test_file", + "agentSeverity": "6", + "CEF_Version": "0", + "cn1": "1", + "cn1Label": "Host ID", + "dvchost": "hostname", + "cn2": "205", + "cn2Label": "Quarantine", + "string": "hello \\\"world\\\"!", + "start": "Nov 08 2020 12:30:00.111 UTC", + "deviceCustomDate1": "Nov 08 2022 12:30:00.111", + "deviceCustomDate1Label": "myDate", + "cfp1": "3.14", + "cfp1Label": "myFloat" + } + ] diff --git a/jc/parsers/cef.py b/jc/parsers/cef.py index 1b78b6954..f00a4746a 100644 --- a/jc/parsers/cef.py +++ b/jc/parsers/cef.py @@ -1,14 +1,20 @@ """jc - JSON Convert CEF string parser -This is a best-effort parser since there are so many variations to CEF -formatting from different vendors. If you require special handling for your -CEF input, you can copy this parser code to the `jc` pluggin directory for -your system and modify it to suit your needs. +This parser conforms to the Microfocus Arcsight CEF specification. If you +require special handling for your CEF input, you can copy this parser code +to the `jc` pluggin directory for your system and modify it to suit your +needs. This parser will accept a single CEF string or multiple CEF string lines. Any text before "CEF" will be ignored. Syslog and CEF escaped characters -(`\\`, `\\"`, `\\]`, `\\|`, `\\n`, `\\r`) are unescaped. To preserve -escaping, use the `--raw` or `raw=True` option in the `parse()` function. +(`\\`, `\\"`, `\\]`, `\\|`, `\\n`, `\\r`) are unescaped. + +Extended fields, as defined in the CEF specification, are relabeled +and the values are converted to their respective types. Extra naive and +UTC epoch timestamps are added where appropriate per the CEF specification. + +To preserve escaping, original keynames, and value types use the `--raw` or +`raw=True` option in the `parse()` function. Usage (cli): @@ -37,6 +43,8 @@ "agentSeverityNum": integer, "CEF_Version": integer, string/integer/float, # [0] + "_epoch": integer, # [1] + "_epoch_utc": integer, # [2] string } ] @@ -44,14 +52,62 @@ [0] Will attempt to convert extended fields to the type specified in the CEF specification. If conversion fails, then the field will remain a string. + [1] Naive calculated epoch timestamp + [2] Timezone-aware calculated epoch timestamp. (UTC only) This value + will be null if a UTC timezone cannot be extracted from the original + timestamp string value. Examples: $ cef | jc --cef -p - [] + [ + { + "deviceVendor": "Trend Micro", + "deviceProduct": "Deep Security Agent", + "deviceVersion": "", + "deviceEventClassId": "4000000", + "name": "Eicar_test_file", + "agentSeverity": 6, + "CEF_Version": 0, + "dvchost": "hostname", + "string": "hello \"world\"!", + "start": "Nov 08 2020 12:30:00.111 UTC", + "start_epoch": 1604867400, + "start_epoch_utc": 1604838600, + "Host_ID": 1, + "Quarantine": 205, + "myDate": "Nov 08 2022 12:30:00.111", + "myDate_epoch": 1667939400, + "myDate_epoch_utc": null, + "myFloat": 3.14, + "agentSeverityString": "Medium", + "agentSeverityNum": 6 + } + ] $ cef | jc --cef -p -r - [] + [ + { + "deviceVendor": "Trend Micro", + "deviceProduct": "Deep Security Agent", + "deviceVersion": "", + "deviceEventClassId": "4000000", + "name": "Eicar_test_file", + "agentSeverity": "6", + "CEF_Version": "0", + "cn1": "1", + "cn1Label": "Host ID", + "dvchost": "hostname", + "cn2": "205", + "cn2Label": "Quarantine", + "string": "hello \\\"world\\\"!", + "start": "Nov 08 2020 12:30:00.111 UTC", + "deviceCustomDate1": "Nov 08 2022 12:30:00.111", + "deviceCustomDate1Label": "myDate", + "cfp1": "3.14", + "cfp1Label": "myFloat" + } + ] """ from typing import List, Dict import re diff --git a/man/jc.1 b/man/jc.1 index 13bc13012..51252d813 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -1,4 +1,4 @@ -.TH jc 1 2022-08-17 1.21.0 "JSON Convert" +.TH jc 1 2022-08-18 1.21.0 "JSON Convert" .SH NAME \fBjc\fP \- JSON Convert JSONifies the output of many CLI tools and file-types .SH SYNOPSIS From 0ee0cb35bc18a2a6ec5ef24ebf4deba9a319f48d Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 18 Aug 2022 11:23:03 -0700 Subject: [PATCH 096/116] add escape chars to doc --- docs/parsers/cef.md | 3 ++- jc/parsers/cef.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/parsers/cef.md b/docs/parsers/cef.md index c59bbb2bd..4ddb70726 100644 --- a/docs/parsers/cef.md +++ b/docs/parsers/cef.md @@ -12,7 +12,8 @@ needs. This parser will accept a single CEF string or multiple CEF string lines. Any text before "CEF" will be ignored. Syslog and CEF escaped characters -(`\\`, `\\"`, `\\]`, `\\|`, `\\n`, `\\r`) are unescaped. +(`\\`, `\\"`, `\\]`, `\\|`, `\\=`, `\\%`, ``\\#, `\\n`, `\\r`) are +unescaped. Extended fields, as defined in the CEF specification, are relabeled and the values are converted to their respective types. Extra naive and diff --git a/jc/parsers/cef.py b/jc/parsers/cef.py index f00a4746a..6a2bdce42 100644 --- a/jc/parsers/cef.py +++ b/jc/parsers/cef.py @@ -7,7 +7,8 @@ This parser will accept a single CEF string or multiple CEF string lines. Any text before "CEF" will be ignored. Syslog and CEF escaped characters -(`\\`, `\\"`, `\\]`, `\\|`, `\\n`, `\\r`) are unescaped. +(`\\`, `\\"`, `\\]`, `\\|`, `\\=`, `\\%`, ``\\#, `\\n`, `\\r`) are +unescaped. Extended fields, as defined in the CEF specification, are relabeled and the values are converted to their respective types. Extra naive and From 7bf6f1b870f438e52b20e990190e1f902bbae71d Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 18 Aug 2022 11:24:49 -0700 Subject: [PATCH 097/116] formatting --- docs/parsers/cef.md | 2 +- jc/parsers/cef.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/parsers/cef.md b/docs/parsers/cef.md index 4ddb70726..abb22cd46 100644 --- a/docs/parsers/cef.md +++ b/docs/parsers/cef.md @@ -12,7 +12,7 @@ needs. This parser will accept a single CEF string or multiple CEF string lines. Any text before "CEF" will be ignored. Syslog and CEF escaped characters -(`\\`, `\\"`, `\\]`, `\\|`, `\\=`, `\\%`, ``\\#, `\\n`, `\\r`) are +(`\\`, `\\"`, `\\]`, `\\|`, `\\=`, `\\%`, `\\#`, `\\n`, and `\\r`) are unescaped. Extended fields, as defined in the CEF specification, are relabeled diff --git a/jc/parsers/cef.py b/jc/parsers/cef.py index 6a2bdce42..4e1403482 100644 --- a/jc/parsers/cef.py +++ b/jc/parsers/cef.py @@ -7,7 +7,7 @@ This parser will accept a single CEF string or multiple CEF string lines. Any text before "CEF" will be ignored. Syslog and CEF escaped characters -(`\\`, `\\"`, `\\]`, `\\|`, `\\=`, `\\%`, ``\\#, `\\n`, `\\r`) are +(`\\`, `\\"`, `\\]`, `\\|`, `\\=`, `\\%`, `\\#`, `\\n`, and `\\r`) are unescaped. Extended fields, as defined in the CEF specification, are relabeled From 89574faef7177c21104949691adcd72e401957c1 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 19 Aug 2022 08:44:23 -0700 Subject: [PATCH 098/116] remove underscore in CEF_Version, doc update, timestamp optimization --- docs/parsers/cef.md | 15 ++++++--------- jc/parsers/cef.py | 29 ++++++++++++++--------------- man/jc.1 | 2 +- 3 files changed, 21 insertions(+), 25 deletions(-) diff --git a/docs/parsers/cef.md b/docs/parsers/cef.md index abb22cd46..cb2753151 100644 --- a/docs/parsers/cef.md +++ b/docs/parsers/cef.md @@ -5,10 +5,7 @@ jc - JSON Convert CEF string parser -This parser conforms to the Microfocus Arcsight CEF specification. If you -require special handling for your CEF input, you can copy this parser code -to the `jc` pluggin directory for your system and modify it to suit your -needs. +This parser conforms to the Microfocus Arcsight CEF specification. This parser will accept a single CEF string or multiple CEF string lines. Any text before "CEF" will be ignored. Syslog and CEF escaped characters @@ -19,8 +16,8 @@ Extended fields, as defined in the CEF specification, are relabeled and the values are converted to their respective types. Extra naive and UTC epoch timestamps are added where appropriate per the CEF specification. -To preserve escaping, original keynames, and value types use the `--raw` or -`raw=True` option in the `parse()` function. +To preserve escaping and original keynames and to prevent type conversions +use the `--raw` or `raw=True` option in the `parse()` function. Usage (cli): @@ -47,7 +44,7 @@ See: https://www.microfocus.com/documentation/arcsight/arcsight-smartconnectors- "agentSeverity": string/integer, "agentSeverityString": string, "agentSeverityNum": integer, - "CEF_Version": integer, + "CEFVersion": integer, string/integer/float, # [0] "_epoch": integer, # [1] "_epoch_utc": integer, # [2] @@ -74,7 +71,7 @@ Examples: "deviceEventClassId": "4000000", "name": "Eicar_test_file", "agentSeverity": 6, - "CEF_Version": 0, + "CEFVersion": 0, "dvchost": "hostname", "string": "hello \"world\"!", "start": "Nov 08 2020 12:30:00.111 UTC", @@ -100,7 +97,7 @@ Examples: "deviceEventClassId": "4000000", "name": "Eicar_test_file", "agentSeverity": "6", - "CEF_Version": "0", + "CEFVersion": "0", "cn1": "1", "cn1Label": "Host ID", "dvchost": "hostname", diff --git a/jc/parsers/cef.py b/jc/parsers/cef.py index 4e1403482..249374d7a 100644 --- a/jc/parsers/cef.py +++ b/jc/parsers/cef.py @@ -1,9 +1,6 @@ """jc - JSON Convert CEF string parser -This parser conforms to the Microfocus Arcsight CEF specification. If you -require special handling for your CEF input, you can copy this parser code -to the `jc` pluggin directory for your system and modify it to suit your -needs. +This parser conforms to the Microfocus Arcsight CEF specification. This parser will accept a single CEF string or multiple CEF string lines. Any text before "CEF" will be ignored. Syslog and CEF escaped characters @@ -14,8 +11,8 @@ and the values are converted to their respective types. Extra naive and UTC epoch timestamps are added where appropriate per the CEF specification. -To preserve escaping, original keynames, and value types use the `--raw` or -`raw=True` option in the `parse()` function. +To preserve escaping and original keynames and to prevent type conversions +use the `--raw` or `raw=True` option in the `parse()` function. Usage (cli): @@ -42,7 +39,7 @@ "agentSeverity": string/integer, "agentSeverityString": string, "agentSeverityNum": integer, - "CEF_Version": integer, + "CEFVersion": integer, string/integer/float, # [0] "_epoch": integer, # [1] "_epoch_utc": integer, # [2] @@ -69,7 +66,7 @@ "deviceEventClassId": "4000000", "name": "Eicar_test_file", "agentSeverity": 6, - "CEF_Version": 0, + "CEFVersion": 0, "dvchost": "hostname", "string": "hello \"world\"!", "start": "Nov 08 2020 12:30:00.111 UTC", @@ -95,7 +92,7 @@ "deviceEventClassId": "4000000", "name": "Eicar_test_file", "agentSeverity": "6", - "CEF_Version": "0", + "CEFVersion": "0", "cn1": "1", "cn1Label": "Host ID", "dvchost": "hostname", @@ -203,7 +200,7 @@ def _pycef_parse(str_input): if cef_start == -1: raise ParseError('Invalid CEF string.') (cef, version) = spl[0][cef_start:].split(':') - values["CEF_Version"] = version + values["CEFVersion"] = version # The ugly, gnarly regex here finds a single key=value pair, # taking into account multiple whitespaces, escaped '=' and '|' @@ -249,7 +246,7 @@ def _process(proc_data: List[Dict]) -> List[Dict]: r'\r': '\r' } - int_list = {'CEF_Version'} + int_list = {'CEFVersion'} severity_map = { None: 'Unknown', @@ -304,7 +301,8 @@ def _process(proc_data: List[Dict]) -> List[Dict]: pass if key in extended_dt: - dt = jc.utils.timestamp(item[key]) + formats = (1400, 1410, 1420, 1430) + dt = jc.utils.timestamp(item[key], formats) item[key + '_epoch'] = dt.naive item[key + '_epoch_utc'] = dt.utc @@ -315,19 +313,20 @@ def _process(proc_data: List[Dict]) -> List[Dict]: if key.endswith('Label'): customlabel = key[:-5] for customfield in custom_fields: + new_name = item[key] # check for normal custom fields if customfield == customlabel: - item[item[key]] = item[customfield] + item[new_name] = item[customfield] cleanup_list.append(customfield) cleanup_list.append(key) # check for datetime objects if customfield == customlabel + '_epoch': - item[item[key] + '_epoch'] = item[customfield] + item[new_name + '_epoch'] = item[customfield] cleanup_list.append(customfield) if customfield == customlabel + '_epoch_utc': - item[item[key] + '_epoch_utc'] = item[customfield] + item[new_name + '_epoch_utc'] = item[customfield] cleanup_list.append(customfield) # cleanup extra custom fields diff --git a/man/jc.1 b/man/jc.1 index 51252d813..4314bdce0 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -1,4 +1,4 @@ -.TH jc 1 2022-08-18 1.21.0 "JSON Convert" +.TH jc 1 2022-08-19 1.21.0 "JSON Convert" .SH NAME \fBjc\fP \- JSON Convert JSONifies the output of many CLI tools and file-types .SH SYNOPSIS From de11ae20a496119562ff9dcaf7bd8b3e39a1907f Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 19 Aug 2022 09:06:46 -0700 Subject: [PATCH 099/116] add git, syslog, and cef timestamp tests --- tests/test_jc_utils.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/test_jc_utils.py b/tests/test_jc_utils.py index 1f8a14b1d..4e934b5f7 100644 --- a/tests/test_jc_utils.py +++ b/tests/test_jc_utils.py @@ -18,6 +18,20 @@ def test_utils_timestamp(self): # C locale format conversion, or date cli command in C locale with non-UTC tz 'Tue Mar 23 16:12:11 2021': {'string': 'Tue Mar 23 16:12:11 2021', 'format': 1000, 'naive': 1616541131, 'utc': None}, 'Tue Mar 23 16:12:11 IST 2021': {'string': 'Tue Mar 23 16:12:11 IST 2021', 'format': 1000, 'naive': 1616541131, 'utc': None}, + # Git date output + 'Thu Mar 5 09:17:40 2020 -0800': {'string': 'Thu Mar 5 09:17:40 2020 -0800', 'format': 1100, 'naive': 1583428660, 'utc': None}, + # ISO Format with UTC (found in syslog 5424) + '2003-10-11T22:14:15.003Z': {'string': '2003-10-11T22:14:15.003Z', 'format': 1300, 'naive': 1065935655, 'utc': 1065910455}, + # ISO Format without TZ (found in syslog 5424) + '2003-10-11T22:14:15.003': {'string': '2003-10-11T22:14:15.003', 'format': 1310, 'naive': 1065935655, 'utc': None}, + # CEF Format with UTC + 'Nov 08 2022 12:30:00.111 UTC': {'string': 'Nov 08 2022 12:30:00.111 UTC', 'format': 1400, 'naive': 1667939400, 'utc': 1667910600}, + # CEF Format without TZ + 'Nov 08 2022 12:30:00.111': {'string': 'Nov 08 2022 12:30:00.111', 'format': 1410, 'naive': 1667939400, 'utc': None}, + # CEF Format with UTC without microseconds + 'Nov 08 2022 12:30:00 UTC': {'string': 'Nov 08 2022 12:30:00 UTC', 'format': 1420, 'naive': 1667939400, 'utc': 1667910600}, + # CEF Format without TZ or microseconds + 'Nov 08 2022 12:30:00': {'string': 'Nov 08 2022 12:30:00', 'format': 1430, 'naive': 1667939400, 'utc': None}, # en_US.UTF-8 local format (found in who cli output) '2021-03-23 00:14': {'string': '2021-03-23 00:14', 'format': 1500, 'naive': 1616483640, 'utc': None}, # Windows english format (found in dir cli output) From 9ddaddfaa09369b59d44ae0f59a5e741ee1d0686 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 19 Aug 2022 20:36:58 -0700 Subject: [PATCH 100/116] add case for timestamp-style datetime fields --- jc/parsers/cef.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/jc/parsers/cef.py b/jc/parsers/cef.py index 249374d7a..f5071b4f5 100644 --- a/jc/parsers/cef.py +++ b/jc/parsers/cef.py @@ -301,10 +301,14 @@ def _process(proc_data: List[Dict]) -> List[Dict]: pass if key in extended_dt: - formats = (1400, 1410, 1420, 1430) - dt = jc.utils.timestamp(item[key], formats) - item[key + '_epoch'] = dt.naive - item[key + '_epoch_utc'] = dt.utc + if re.match(r'\d{10,13}', item[key]): + item[key + '_epoch'] = int(item[key][:10]) + item[key + '_epoch_utc'] = None + else: + formats = (1400, 1410, 1420, 1430) + dt = jc.utils.timestamp(item[key], formats) + item[key + '_epoch'] = dt.naive + item[key + '_epoch_utc'] = dt.utc # Process custom field labels (adapted from pycef library) cleanup_list = [] From 985d7d28edecb1203dff2675f0d445557f265c30 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 19 Aug 2022 21:36:15 -0700 Subject: [PATCH 101/116] add -n support (ipv4) --- jc/parsers/traceroute.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/jc/parsers/traceroute.py b/jc/parsers/traceroute.py index 937720123..924d95caa 100644 --- a/jc/parsers/traceroute.py +++ b/jc/parsers/traceroute.py @@ -122,7 +122,7 @@ class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.5' + version = '1.6' description = '`traceroute` and `traceroute6` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -164,6 +164,7 @@ class info(): RE_HEADER = re.compile(r'(\S+)\s+\((\d+\.\d+\.\d+\.\d+|[0-9a-fA-F:]+)\)') RE_PROBE_NAME_IP = re.compile(r'(\S+)\s+\((\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|[0-9a-fA-F:]+)\)+') +RE_PROBE_IP_ONLY = re.compile(r'(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})\s+([^\(])') RE_PROBE_BSD_IPV6 = re.compile(r'\b(?:[A-Fa-f0-9]{1,4}:){7}[A-Fa-f0-9]{1,4}\b') RE_HOP = re.compile(r'^\s*(\d+)?\s+(.+)$') RE_PROBE_ASN = re.compile(r'\[AS(\d+)\]') @@ -274,8 +275,12 @@ def _loads(data): probe_asn = None probe_name_ip_match = RE_PROBE_NAME_IP.search(hop_string) + probe_ip_only_match = RE_PROBE_IP_ONLY.search(hop_string) probe_bsd_ipv6_match = RE_PROBE_BSD_IPV6.search(hop_string) - if probe_name_ip_match: + if probe_ip_only_match: + probe_name = None + probe_ip = probe_ip_only_match.group(1) + elif probe_name_ip_match: probe_name = probe_name_ip_match.group(1) probe_ip = probe_name_ip_match.group(2) elif probe_bsd_ipv6_match: From 23ad50159538a57963e5d2942778e22e3abc1ac1 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sat, 20 Aug 2022 09:48:38 -0700 Subject: [PATCH 102/116] add -n support for ipv6 --- jc/parsers/traceroute.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/jc/parsers/traceroute.py b/jc/parsers/traceroute.py index 924d95caa..e34cf10dd 100644 --- a/jc/parsers/traceroute.py +++ b/jc/parsers/traceroute.py @@ -165,6 +165,7 @@ class info(): RE_HEADER = re.compile(r'(\S+)\s+\((\d+\.\d+\.\d+\.\d+|[0-9a-fA-F:]+)\)') RE_PROBE_NAME_IP = re.compile(r'(\S+)\s+\((\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|[0-9a-fA-F:]+)\)+') RE_PROBE_IP_ONLY = re.compile(r'(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})\s+([^\(])') +RE_PROBE_IPV6_ONLY = re.compile(r'(([a-f0-9:]+:+)+[a-f0-9]+)') RE_PROBE_BSD_IPV6 = re.compile(r'\b(?:[A-Fa-f0-9]{1,4}:){7}[A-Fa-f0-9]{1,4}\b') RE_HOP = re.compile(r'^\s*(\d+)?\s+(.+)$') RE_PROBE_ASN = re.compile(r'\[AS(\d+)\]') @@ -277,6 +278,7 @@ def _loads(data): probe_name_ip_match = RE_PROBE_NAME_IP.search(hop_string) probe_ip_only_match = RE_PROBE_IP_ONLY.search(hop_string) probe_bsd_ipv6_match = RE_PROBE_BSD_IPV6.search(hop_string) + probe_ipv6_only_match = RE_PROBE_IPV6_ONLY.search(hop_string) if probe_ip_only_match: probe_name = None probe_ip = probe_ip_only_match.group(1) @@ -286,6 +288,9 @@ def _loads(data): elif probe_bsd_ipv6_match: probe_name = None probe_ip = probe_bsd_ipv6_match.group(0) + elif probe_ipv6_only_match: + probe_name = None + probe_ip = probe_ipv6_only_match.group(1) else: probe_name = None probe_ip = None From 6d5c72b54b9bf5d3a9d3cb73464a70395b0919ea Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sat, 20 Aug 2022 10:58:13 -0700 Subject: [PATCH 103/116] add raid5 and container tests --- .../generic/mdadm-examine-container1.json | 1 + .../generic/mdadm-examine-container1.out | 27 +++ .../mdadm-examine-container2-dev1.json | 1 + .../generic/mdadm-examine-container2-dev1.out | 39 ++++ .../mdadm-examine-container2-dev2.json | 1 + .../generic/mdadm-examine-container2-dev2.out | 39 ++++ .../generic/mdadm-examine-raid5-homehost.json | 1 + .../generic/mdadm-examine-raid5-homehost.out | 28 +++ .../generic/mdadm-examine-raid5-meta09.json | 1 + .../generic/mdadm-examine-raid5-meta09.out | 30 ++++ .../generic/mdadm-examine-raid5-ok.json | 1 + .../generic/mdadm-examine-raid5-ok.out | 28 +++ .../mdadm-query-container1-member.json | 1 + .../generic/mdadm-query-container1-member.out | 23 +++ .../generic/mdadm-query-container1-root.json | 1 + .../generic/mdadm-query-container1-root.out | 18 ++ .../mdadm-query-container2-member.json | 1 + .../generic/mdadm-query-container2-member.out | 23 +++ .../generic/mdadm-query-container2-root.json | 1 + .../generic/mdadm-query-container2-root.out | 18 ++ .../generic/mdadm-query-raid5-homehost.json | 1 + .../generic/mdadm-query-raid5-homehost.out | 30 ++++ .../generic/mdadm-query-raid5-meta09.json | 1 + .../generic/mdadm-query-raid5-meta09.out | 30 ++++ .../generic/mdadm-query-raid5-ok.json | 1 + .../fixtures/generic/mdadm-query-raid5-ok.out | 30 ++++ tests/test_mdadm.py | 169 ++++++++++++++++++ 27 files changed, 545 insertions(+) create mode 100644 tests/fixtures/generic/mdadm-examine-container1.json create mode 100644 tests/fixtures/generic/mdadm-examine-container1.out create mode 100644 tests/fixtures/generic/mdadm-examine-container2-dev1.json create mode 100644 tests/fixtures/generic/mdadm-examine-container2-dev1.out create mode 100644 tests/fixtures/generic/mdadm-examine-container2-dev2.json create mode 100644 tests/fixtures/generic/mdadm-examine-container2-dev2.out create mode 100644 tests/fixtures/generic/mdadm-examine-raid5-homehost.json create mode 100644 tests/fixtures/generic/mdadm-examine-raid5-homehost.out create mode 100644 tests/fixtures/generic/mdadm-examine-raid5-meta09.json create mode 100644 tests/fixtures/generic/mdadm-examine-raid5-meta09.out create mode 100644 tests/fixtures/generic/mdadm-examine-raid5-ok.json create mode 100644 tests/fixtures/generic/mdadm-examine-raid5-ok.out create mode 100644 tests/fixtures/generic/mdadm-query-container1-member.json create mode 100644 tests/fixtures/generic/mdadm-query-container1-member.out create mode 100644 tests/fixtures/generic/mdadm-query-container1-root.json create mode 100644 tests/fixtures/generic/mdadm-query-container1-root.out create mode 100644 tests/fixtures/generic/mdadm-query-container2-member.json create mode 100644 tests/fixtures/generic/mdadm-query-container2-member.out create mode 100644 tests/fixtures/generic/mdadm-query-container2-root.json create mode 100644 tests/fixtures/generic/mdadm-query-container2-root.out create mode 100644 tests/fixtures/generic/mdadm-query-raid5-homehost.json create mode 100644 tests/fixtures/generic/mdadm-query-raid5-homehost.out create mode 100644 tests/fixtures/generic/mdadm-query-raid5-meta09.json create mode 100644 tests/fixtures/generic/mdadm-query-raid5-meta09.out create mode 100644 tests/fixtures/generic/mdadm-query-raid5-ok.json create mode 100644 tests/fixtures/generic/mdadm-query-raid5-ok.out diff --git a/tests/fixtures/generic/mdadm-examine-container1.json b/tests/fixtures/generic/mdadm-examine-container1.json new file mode 100644 index 000000000..6f8b52128 --- /dev/null +++ b/tests/fixtures/generic/mdadm-examine-container1.json @@ -0,0 +1 @@ +{"device":"/dev/vda","magic":"de11de11","version":"01.02.00","controller_guid":"4C696E75:782D4D44:20202020:20202073:79737265:73637565","container_guid":"4C696E75:782D4D44:DEADBEEF:00000000:502D01BA:FB6D4FCB","seq":"00000008","redundant_hdr":"yes","virtual_disks":1,"vd_guid_0":"4C696E75:782D4D44:DEADBEEF:00000000:502D01DF:DBFB2605","unit_0":"127","state_0":"Optimal, Consistent","init_state_0":"Fully Initialised","access_0":"Read/Write","name_0":"vol0","raid_devices_0":"2 (1@0K 0@0K)","raid_level_0":"RAID1","device_size_0":"786432","array_size_0":"786432","physical_disks":1023,"device_table":[{"Number":0,"RefNo":"d32b14d7","Size":"786432K","Device":"/dev/vda","Type/State":"active/Online"},{"Number":1,"RefNo":"886a9f58","Size":"913408K","Device":null,"Type/State":"active/Online"}]} diff --git a/tests/fixtures/generic/mdadm-examine-container1.out b/tests/fixtures/generic/mdadm-examine-container1.out new file mode 100644 index 000000000..d2b8e4bd7 --- /dev/null +++ b/tests/fixtures/generic/mdadm-examine-container1.out @@ -0,0 +1,27 @@ +/dev/vda: + Magic : de11de11 + Version : 01.02.00 +Controller GUID : 4C696E75:782D4D44:20202020:20202073:79737265:73637565 + (Linux-MD sysrescue) + Container GUID : 4C696E75:782D4D44:DEADBEEF:00000000:502D01BA:FB6D4FCB + (Linux-MD 08/16/22 16:20:42) + Seq : 00000008 + Redundant hdr : yes + Virtual Disks : 1 + + VD GUID[0] : 4C696E75:782D4D44:DEADBEEF:00000000:502D01DF:DBFB2605 + (Linux-MD 08/16/22 16:21:19) + unit[0] : 127 + state[0] : Optimal, Consistent + init state[0] : Fully Initialised + access[0] : Read/Write + Name[0] : vol0 + Raid Devices[0] : 2 (1@0K 0@0K) + Raid Level[0] : RAID1 + Device Size[0] : 786432 + Array Size[0] : 786432 + + Physical Disks : 1023 + Number RefNo Size Device Type/State + 0 d32b14d7 786432K /dev/vda active/Online + 1 886a9f58 913408K active/Online diff --git a/tests/fixtures/generic/mdadm-examine-container2-dev1.json b/tests/fixtures/generic/mdadm-examine-container2-dev1.json new file mode 100644 index 000000000..7738e947e --- /dev/null +++ b/tests/fixtures/generic/mdadm-examine-container2-dev1.json @@ -0,0 +1 @@ +{"device":"/dev/vda","magic":"de11de11","version":"01.02.00","controller_guid":"4C696E75:782D4D44:20202020:20202073:79737265:73637565","container_guid":"4C696E75:782D4D44:DEADBEEF:00000000:502D04BE:255CF1BB","seq":"0000000e","redundant_hdr":"yes","virtual_disks":2,"vd_guid_0":"4C696E75:782D4D44:DEADBEEF:00000000:502D04C9:67A05911","unit_0":"127","state_0":"Optimal, Consistent","init_state_0":"Fully Initialised","access_0":"Read/Write","name_0":"vol0","raid_devices_0":"2 (1@0K 0@0K)","raid_level_0":"RAID1","device_size_0":"1024","array_size_0":"1024","vd_guid_1":"4C696E75:782D4D44:DEADBEEF:00000000:502D04D0:BA95BF7D","unit_1":"126","state_1":"Optimal, Consistent","init_state_1":"Fully Initialised","access_1":"Read/Write","name_1":"vol1","raid_devices_1":"2 (1@1024K 0@1024K)","raid_level_1":"RAID1","device_size_1":"1024","array_size_1":"1024","physical_disks":1023,"device_table":[{"Number":0,"RefNo":"33552859","Size":"786432K","Device":"/dev/vda","Type/State":"active/Online"},{"Number":1,"RefNo":"1a6ba673","Size":"913408K","Device":null,"Type/State":"active/Online"}]} diff --git a/tests/fixtures/generic/mdadm-examine-container2-dev1.out b/tests/fixtures/generic/mdadm-examine-container2-dev1.out new file mode 100644 index 000000000..5061b8c9e --- /dev/null +++ b/tests/fixtures/generic/mdadm-examine-container2-dev1.out @@ -0,0 +1,39 @@ +/dev/vda: + Magic : de11de11 + Version : 01.02.00 +Controller GUID : 4C696E75:782D4D44:20202020:20202073:79737265:73637565 + (Linux-MD sysrescue) + Container GUID : 4C696E75:782D4D44:DEADBEEF:00000000:502D04BE:255CF1BB + (Linux-MD 08/16/22 16:33:34) + Seq : 0000000e + Redundant hdr : yes + Virtual Disks : 2 + + VD GUID[0] : 4C696E75:782D4D44:DEADBEEF:00000000:502D04C9:67A05911 + (Linux-MD 08/16/22 16:33:45) + unit[0] : 127 + state[0] : Optimal, Consistent + init state[0] : Fully Initialised + access[0] : Read/Write + Name[0] : vol0 + Raid Devices[0] : 2 (1@0K 0@0K) + Raid Level[0] : RAID1 + Device Size[0] : 1024 + Array Size[0] : 1024 + + VD GUID[1] : 4C696E75:782D4D44:DEADBEEF:00000000:502D04D0:BA95BF7D + (Linux-MD 08/16/22 16:33:52) + unit[1] : 126 + state[1] : Optimal, Consistent + init state[1] : Fully Initialised + access[1] : Read/Write + Name[1] : vol1 + Raid Devices[1] : 2 (1@1024K 0@1024K) + Raid Level[1] : RAID1 + Device Size[1] : 1024 + Array Size[1] : 1024 + + Physical Disks : 1023 + Number RefNo Size Device Type/State + 0 33552859 786432K /dev/vda active/Online + 1 1a6ba673 913408K active/Online diff --git a/tests/fixtures/generic/mdadm-examine-container2-dev2.json b/tests/fixtures/generic/mdadm-examine-container2-dev2.json new file mode 100644 index 000000000..9d1b257d4 --- /dev/null +++ b/tests/fixtures/generic/mdadm-examine-container2-dev2.json @@ -0,0 +1 @@ +{"device":"/dev/vdb","magic":"de11de11","version":"01.02.00","controller_guid":"4C696E75:782D4D44:20202020:20202073:79737265:73637565","container_guid":"4C696E75:782D4D44:DEADBEEF:00000000:502D04BE:255CF1BB","seq":"0000000e","redundant_hdr":"yes","virtual_disks":2,"vd_guid_0":"4C696E75:782D4D44:DEADBEEF:00000000:502D04C9:67A05911","unit_0":"127","state_0":"Optimal, Consistent","init_state_0":"Fully Initialised","access_0":"Read/Write","name_0":"vol0","raid_devices_0":"2 (1@0K 0@0K)","raid_level_0":"RAID1","device_size_0":"1024","array_size_0":"1024","vd_guid_1":"4C696E75:782D4D44:DEADBEEF:00000000:502D04D0:BA95BF7D","unit_1":"126","state_1":"Optimal, Consistent","init_state_1":"Fully Initialised","access_1":"Read/Write","name_1":"vol1","raid_devices_1":"2 (1@1024K 0@1024K)","raid_level_1":"RAID1","device_size_1":"1024","array_size_1":"1024","physical_disks":1023,"device_table":[{"Number":0,"RefNo":"33552859","Size":"786432K","Device":null,"Type/State":"active/Online"},{"Number":1,"RefNo":"1a6ba673","Size":"913408K","Device":"/dev/vdb","Type/State":"active/Online"}]} diff --git a/tests/fixtures/generic/mdadm-examine-container2-dev2.out b/tests/fixtures/generic/mdadm-examine-container2-dev2.out new file mode 100644 index 000000000..75f440868 --- /dev/null +++ b/tests/fixtures/generic/mdadm-examine-container2-dev2.out @@ -0,0 +1,39 @@ +/dev/vdb: + Magic : de11de11 + Version : 01.02.00 +Controller GUID : 4C696E75:782D4D44:20202020:20202073:79737265:73637565 + (Linux-MD sysrescue) + Container GUID : 4C696E75:782D4D44:DEADBEEF:00000000:502D04BE:255CF1BB + (Linux-MD 08/16/22 16:33:34) + Seq : 0000000e + Redundant hdr : yes + Virtual Disks : 2 + + VD GUID[0] : 4C696E75:782D4D44:DEADBEEF:00000000:502D04C9:67A05911 + (Linux-MD 08/16/22 16:33:45) + unit[0] : 127 + state[0] : Optimal, Consistent + init state[0] : Fully Initialised + access[0] : Read/Write + Name[0] : vol0 + Raid Devices[0] : 2 (1@0K 0@0K) + Raid Level[0] : RAID1 + Device Size[0] : 1024 + Array Size[0] : 1024 + + VD GUID[1] : 4C696E75:782D4D44:DEADBEEF:00000000:502D04D0:BA95BF7D + (Linux-MD 08/16/22 16:33:52) + unit[1] : 126 + state[1] : Optimal, Consistent + init state[1] : Fully Initialised + access[1] : Read/Write + Name[1] : vol1 + Raid Devices[1] : 2 (1@1024K 0@1024K) + Raid Level[1] : RAID1 + Device Size[1] : 1024 + Array Size[1] : 1024 + + Physical Disks : 1023 + Number RefNo Size Device Type/State + 0 33552859 786432K active/Online + 1 1a6ba673 913408K /dev/vdb active/Online diff --git a/tests/fixtures/generic/mdadm-examine-raid5-homehost.json b/tests/fixtures/generic/mdadm-examine-raid5-homehost.json new file mode 100644 index 000000000..26b697b57 --- /dev/null +++ b/tests/fixtures/generic/mdadm-examine-raid5-homehost.json @@ -0,0 +1 @@ +{"device":"/dev/vda1","magic":"a92b4efc","version":"1.2","feature_map":"0x0","array_uuid":"07bcc229:6af63526:b1fc14ff:af3524ef","name":"foohost:0","creation_time":"Tue Aug 16 17:02:01 2022","raid_level":"raid5","raid_devices":3,"avail_dev_size":"405504 sectors (198.00 MiB 207.62 MB)","array_size":"405504 KiB (396.00 MiB 415.24 MB)","data_offset":4096,"super_offset":8,"unused_space":"before=4016 sectors, after=0 sectors","state":"clean","device_uuid":"28848fbc:1648c806:004ddc4a:7820c3fc","update_time":"Tue Aug 16 17:02:05 2022","bad_block_log":"512 entries available at offset 16 sectors","checksum":"5b482818 - correct","events":"18","layout":"left-symmetric","chunk_size":"512K","device_role":"Active device 0","array_state":"AAA ('A' == active, '.' == missing, 'R' == replacing)","array_size_num":405504,"avail_dev_size_num":405504,"unused_space_before":4016,"unused_space_after":0,"name_val":"foohost:0","chunk_size_num":512,"events_num":18,"checksum_val":"5b482818","checksum_state":"correct","state_list":["clean"],"array_state_list":["active","active","active"],"creation_time_epoch":1660694521,"update_time_epoch":1660694525} diff --git a/tests/fixtures/generic/mdadm-examine-raid5-homehost.out b/tests/fixtures/generic/mdadm-examine-raid5-homehost.out new file mode 100644 index 000000000..e77187fea --- /dev/null +++ b/tests/fixtures/generic/mdadm-examine-raid5-homehost.out @@ -0,0 +1,28 @@ +/dev/vda1: + Magic : a92b4efc + Version : 1.2 + Feature Map : 0x0 + Array UUID : 07bcc229:6af63526:b1fc14ff:af3524ef + Name : foohost:0 + Creation Time : Tue Aug 16 17:02:01 2022 + Raid Level : raid5 + Raid Devices : 3 + + Avail Dev Size : 405504 sectors (198.00 MiB 207.62 MB) + Array Size : 405504 KiB (396.00 MiB 415.24 MB) + Data Offset : 4096 sectors + Super Offset : 8 sectors + Unused Space : before=4016 sectors, after=0 sectors + State : clean + Device UUID : 28848fbc:1648c806:004ddc4a:7820c3fc + + Update Time : Tue Aug 16 17:02:05 2022 + Bad Block Log : 512 entries available at offset 16 sectors + Checksum : 5b482818 - correct + Events : 18 + + Layout : left-symmetric + Chunk Size : 512K + + Device Role : Active device 0 + Array State : AAA ('A' == active, '.' == missing, 'R' == replacing) diff --git a/tests/fixtures/generic/mdadm-examine-raid5-meta09.json b/tests/fixtures/generic/mdadm-examine-raid5-meta09.json new file mode 100644 index 000000000..1be34c388 --- /dev/null +++ b/tests/fixtures/generic/mdadm-examine-raid5-meta09.json @@ -0,0 +1 @@ +{"device":"/dev/vda1","magic":"a92b4efc","version":"0.90.00","uuid":"4de9c6a6:ce05eada:96fc04a8:108a4af7 (local to host sysrescue)","creation_time":"Tue Aug 16 17:05:02 2022","raid_level":"raid5","used_dev_size":"204288 (199.50 MiB 209.19 MB)","array_size":"408576 (399.00 MiB 418.38 MB)","raid_devices":3,"total_devices":3,"preferred_minor":0,"update_time":"Tue Aug 16 17:05:06 2022","state":"clean","active_devices":3,"working_devices":3,"failed_devices":0,"spare_devices":0,"checksum":"32a3d70f - correct","events":"19","layout":"left-symmetric","chunk_size":"512K","device_table":[{"number":0,"major":254,"minor":1,"state":["active","sync"],"device":"/dev/vda1","raid_device":0},{"number":0,"major":254,"minor":1,"state":["active","sync"],"device":"/dev/vda1","raid_device":0},{"number":11,"major":254,"minor":2,"state":["active","sync"],"device":"/dev/vda2","raid_device":1},{"number":22,"major":254,"minor":3,"state":["active","sync"],"device":"/dev/vda3","raid_device":2}],"array_size_num":408576,"used_dev_size_num":204288,"uuid_val":"4de9c6a6:ce05eada:96fc04a8:108a4af7","homehost":"sysrescue","chunk_size_num":512,"events_num":19,"checksum_val":"32a3d70f","checksum_state":"correct","state_list":["clean"],"creation_time_epoch":1660694702,"update_time_epoch":1660694706} diff --git a/tests/fixtures/generic/mdadm-examine-raid5-meta09.out b/tests/fixtures/generic/mdadm-examine-raid5-meta09.out new file mode 100644 index 000000000..49fdd3c7e --- /dev/null +++ b/tests/fixtures/generic/mdadm-examine-raid5-meta09.out @@ -0,0 +1,30 @@ +/dev/vda1: + Magic : a92b4efc + Version : 0.90.00 + UUID : 4de9c6a6:ce05eada:96fc04a8:108a4af7 (local to host sysrescue) + Creation Time : Tue Aug 16 17:05:02 2022 + Raid Level : raid5 + Used Dev Size : 204288 (199.50 MiB 209.19 MB) + Array Size : 408576 (399.00 MiB 418.38 MB) + Raid Devices : 3 + Total Devices : 3 +Preferred Minor : 0 + + Update Time : Tue Aug 16 17:05:06 2022 + State : clean + Active Devices : 3 +Working Devices : 3 + Failed Devices : 0 + Spare Devices : 0 + Checksum : 32a3d70f - correct + Events : 19 + + Layout : left-symmetric + Chunk Size : 512K + + Number Major Minor RaidDevice State +this 0 254 1 0 active sync /dev/vda1 + + 0 0 254 1 0 active sync /dev/vda1 + 1 1 254 2 1 active sync /dev/vda2 + 2 2 254 3 2 active sync /dev/vda3 diff --git a/tests/fixtures/generic/mdadm-examine-raid5-ok.json b/tests/fixtures/generic/mdadm-examine-raid5-ok.json new file mode 100644 index 000000000..da026e979 --- /dev/null +++ b/tests/fixtures/generic/mdadm-examine-raid5-ok.json @@ -0,0 +1 @@ +{"device":"/dev/vda1","magic":"a92b4efc","version":"1.2","feature_map":"0x0","array_uuid":"4d798d62:adfb6a9f:cbe9a393:b79cd3f4","name":"sysrescue:0 (local to host sysrescue)","creation_time":"Tue Aug 16 16:59:34 2022","raid_level":"raid5","raid_devices":3,"avail_dev_size":"405504 sectors (198.00 MiB 207.62 MB)","array_size":"405504 KiB (396.00 MiB 415.24 MB)","data_offset":4096,"super_offset":8,"unused_space":"before=4016 sectors, after=0 sectors","state":"clean","device_uuid":"56dec54e:4d8b51c1:a7a07f77:de3be376","update_time":"Tue Aug 16 16:59:39 2022","bad_block_log":"512 entries available at offset 16 sectors","checksum":"e02c8e48 - correct","events":"18","layout":"left-symmetric","chunk_size":"512K","device_role":"Active device 0","array_state":"AAA ('A' == active, '.' == missing, 'R' == replacing)","array_size_num":405504,"avail_dev_size_num":405504,"unused_space_before":4016,"unused_space_after":0,"name_val":"sysrescue:0","homehost":"sysrescue","chunk_size_num":512,"events_num":18,"checksum_val":"e02c8e48","checksum_state":"correct","state_list":["clean"],"array_state_list":["active","active","active"],"creation_time_epoch":1660694374,"update_time_epoch":1660694379} diff --git a/tests/fixtures/generic/mdadm-examine-raid5-ok.out b/tests/fixtures/generic/mdadm-examine-raid5-ok.out new file mode 100644 index 000000000..28be27451 --- /dev/null +++ b/tests/fixtures/generic/mdadm-examine-raid5-ok.out @@ -0,0 +1,28 @@ +/dev/vda1: + Magic : a92b4efc + Version : 1.2 + Feature Map : 0x0 + Array UUID : 4d798d62:adfb6a9f:cbe9a393:b79cd3f4 + Name : sysrescue:0 (local to host sysrescue) + Creation Time : Tue Aug 16 16:59:34 2022 + Raid Level : raid5 + Raid Devices : 3 + + Avail Dev Size : 405504 sectors (198.00 MiB 207.62 MB) + Array Size : 405504 KiB (396.00 MiB 415.24 MB) + Data Offset : 4096 sectors + Super Offset : 8 sectors + Unused Space : before=4016 sectors, after=0 sectors + State : clean + Device UUID : 56dec54e:4d8b51c1:a7a07f77:de3be376 + + Update Time : Tue Aug 16 16:59:39 2022 + Bad Block Log : 512 entries available at offset 16 sectors + Checksum : e02c8e48 - correct + Events : 18 + + Layout : left-symmetric + Chunk Size : 512K + + Device Role : Active device 0 + Array State : AAA ('A' == active, '.' == missing, 'R' == replacing) diff --git a/tests/fixtures/generic/mdadm-query-container1-member.json b/tests/fixtures/generic/mdadm-query-container1-member.json new file mode 100644 index 000000000..94b7337a6 --- /dev/null +++ b/tests/fixtures/generic/mdadm-query-container1-member.json @@ -0,0 +1 @@ +{"device":"/dev/md127","container":"/dev/md0, member 0","raid_level":"raid1","array_size":"786432 (768.00 MiB 805.31 MB)","used_dev_size":"786432 (768.00 MiB 805.31 MB)","raid_devices":2,"total_devices":2,"state":"clean","active_devices":2,"working_devices":2,"failed_devices":0,"consistency_policy":"resync","container_guid":"4C696E75:782D4D44:DEADBEEF:00000000:502D01BA:FB6D4FCB","seq":"00000008","virtual_disks":1,"device_table":[{"number":0,"major":254,"minor":16,"state":["active","sync"],"device":"/dev/vdb","raid_device":0},{"number":1,"major":254,"minor":0,"state":["active","sync"],"device":"/dev/vda","raid_device":1}],"array_size_num":786432,"used_dev_size_num":786432,"container_dev":"/dev/md0","container_member":0,"state_list":["clean"]} diff --git a/tests/fixtures/generic/mdadm-query-container1-member.out b/tests/fixtures/generic/mdadm-query-container1-member.out new file mode 100644 index 000000000..8b4122d5a --- /dev/null +++ b/tests/fixtures/generic/mdadm-query-container1-member.out @@ -0,0 +1,23 @@ +/dev/md127: + Container : /dev/md0, member 0 + Raid Level : raid1 + Array Size : 786432 (768.00 MiB 805.31 MB) + Used Dev Size : 786432 (768.00 MiB 805.31 MB) + Raid Devices : 2 + Total Devices : 2 + + State : clean + Active Devices : 2 + Working Devices : 2 + Failed Devices : 0 + +Consistency Policy : resync + + Container GUID : 4C696E75:782D4D44:DEADBEEF:00000000:502D01BA:FB6D4FCB + (Linux-MD 08/16/22 16:20:42) + Seq : 00000008 + Virtual Disks : 1 + + Number Major Minor RaidDevice State + 0 254 16 0 active sync /dev/vdb + 1 254 0 1 active sync /dev/vda diff --git a/tests/fixtures/generic/mdadm-query-container1-root.json b/tests/fixtures/generic/mdadm-query-container1-root.json new file mode 100644 index 000000000..0f9884cde --- /dev/null +++ b/tests/fixtures/generic/mdadm-query-container1-root.json @@ -0,0 +1 @@ +{"device":"/dev/md0","version":"ddf","raid_level":"container","total_devices":2,"working_devices":2,"container_guid":"4C696E75:782D4D44:DEADBEEF:00000000:502D01BA:FB6D4FCB","seq":"00000008","virtual_disks":1,"member_arrays":"/dev/md/vol0","device_table":[{"number":null,"major":254,"minor":0,"device":"/dev/vda","raid_device":null},{"number":null,"major":254,"minor":16,"device":"/dev/vdb","raid_device":null}],"member_arrays_list":["/dev/md/vol0"]} diff --git a/tests/fixtures/generic/mdadm-query-container1-root.out b/tests/fixtures/generic/mdadm-query-container1-root.out new file mode 100644 index 000000000..9630c33bc --- /dev/null +++ b/tests/fixtures/generic/mdadm-query-container1-root.out @@ -0,0 +1,18 @@ +/dev/md0: + Version : ddf + Raid Level : container + Total Devices : 2 + + Working Devices : 2 + + Container GUID : 4C696E75:782D4D44:DEADBEEF:00000000:502D01BA:FB6D4FCB + (Linux-MD 08/16/22 16:20:42) + Seq : 00000008 + Virtual Disks : 1 + + Member Arrays : /dev/md/vol0 + + Number Major Minor RaidDevice + + - 254 0 - /dev/vda + - 254 16 - /dev/vdb diff --git a/tests/fixtures/generic/mdadm-query-container2-member.json b/tests/fixtures/generic/mdadm-query-container2-member.json new file mode 100644 index 000000000..80a434c60 --- /dev/null +++ b/tests/fixtures/generic/mdadm-query-container2-member.json @@ -0,0 +1 @@ +{"device":"/dev/md126","container":"/dev/md0, member 1","raid_level":"raid1","array_size":"1024","used_dev_size":"1024","raid_devices":2,"total_devices":2,"state":"clean","active_devices":2,"working_devices":2,"failed_devices":0,"consistency_policy":"resync","container_guid":"4C696E75:782D4D44:DEADBEEF:00000000:502D04BE:255CF1BB","seq":"0000000e","virtual_disks":2,"device_table":[{"number":0,"major":254,"minor":16,"state":["active","sync"],"device":"/dev/vdb","raid_device":0},{"number":1,"major":254,"minor":0,"state":["active","sync"],"device":"/dev/vda","raid_device":1}],"array_size_num":1024,"used_dev_size_num":1024,"container_dev":"/dev/md0","container_member":1,"state_list":["clean"]} diff --git a/tests/fixtures/generic/mdadm-query-container2-member.out b/tests/fixtures/generic/mdadm-query-container2-member.out new file mode 100644 index 000000000..ad913c816 --- /dev/null +++ b/tests/fixtures/generic/mdadm-query-container2-member.out @@ -0,0 +1,23 @@ +/dev/md126: + Container : /dev/md0, member 1 + Raid Level : raid1 + Array Size : 1024 + Used Dev Size : 1024 + Raid Devices : 2 + Total Devices : 2 + + State : clean + Active Devices : 2 + Working Devices : 2 + Failed Devices : 0 + +Consistency Policy : resync + + Container GUID : 4C696E75:782D4D44:DEADBEEF:00000000:502D04BE:255CF1BB + (Linux-MD 08/16/22 16:33:34) + Seq : 0000000e + Virtual Disks : 2 + + Number Major Minor RaidDevice State + 0 254 16 0 active sync /dev/vdb + 1 254 0 1 active sync /dev/vda diff --git a/tests/fixtures/generic/mdadm-query-container2-root.json b/tests/fixtures/generic/mdadm-query-container2-root.json new file mode 100644 index 000000000..f3532ce7c --- /dev/null +++ b/tests/fixtures/generic/mdadm-query-container2-root.json @@ -0,0 +1 @@ +{"device":"/dev/md0","version":"ddf","raid_level":"container","total_devices":2,"working_devices":2,"container_guid":"4C696E75:782D4D44:DEADBEEF:00000000:502D04BE:255CF1BB","seq":"0000000e","virtual_disks":2,"member_arrays":"/dev/md/vol0 /dev/md/vol1","device_table":[{"number":null,"major":254,"minor":0,"device":"/dev/vda","raid_device":null},{"number":null,"major":254,"minor":16,"device":"/dev/vdb","raid_device":null}],"member_arrays_list":["/dev/md/vol0","/dev/md/vol1"]} diff --git a/tests/fixtures/generic/mdadm-query-container2-root.out b/tests/fixtures/generic/mdadm-query-container2-root.out new file mode 100644 index 000000000..cb8faef59 --- /dev/null +++ b/tests/fixtures/generic/mdadm-query-container2-root.out @@ -0,0 +1,18 @@ +/dev/md0: + Version : ddf + Raid Level : container + Total Devices : 2 + + Working Devices : 2 + + Container GUID : 4C696E75:782D4D44:DEADBEEF:00000000:502D04BE:255CF1BB + (Linux-MD 08/16/22 16:33:34) + Seq : 0000000e + Virtual Disks : 2 + + Member Arrays : /dev/md/vol0 /dev/md/vol1 + + Number Major Minor RaidDevice + + - 254 0 - /dev/vda + - 254 16 - /dev/vdb diff --git a/tests/fixtures/generic/mdadm-query-raid5-homehost.json b/tests/fixtures/generic/mdadm-query-raid5-homehost.json new file mode 100644 index 000000000..48a826000 --- /dev/null +++ b/tests/fixtures/generic/mdadm-query-raid5-homehost.json @@ -0,0 +1 @@ +{"device":"/dev/md0","version":"1.2","creation_time":"Tue Aug 16 17:02:01 2022","raid_level":"raid5","array_size":"405504 (396.00 MiB 415.24 MB)","used_dev_size":"202752 (198.00 MiB 207.62 MB)","raid_devices":3,"total_devices":3,"persistence":"Superblock is persistent","update_time":"Tue Aug 16 17:02:05 2022","state":"clean","active_devices":3,"working_devices":3,"failed_devices":0,"spare_devices":0,"layout":"left-symmetric","chunk_size":"512K","consistency_policy":"resync","name":"foohost:0","uuid":"07bcc229:6af63526:b1fc14ff:af3524ef","events":"18","device_table":[{"number":0,"major":254,"minor":1,"state":["active","sync"],"device":"/dev/vda1","raid_device":0},{"number":1,"major":254,"minor":2,"state":["active","sync"],"device":"/dev/vda2","raid_device":1},{"number":3,"major":254,"minor":3,"state":["active","sync"],"device":"/dev/vda3","raid_device":2}],"array_size_num":405504,"used_dev_size_num":202752,"name_val":"foohost:0","uuid_val":"07bcc229:6af63526:b1fc14ff:af3524ef","chunk_size_num":512,"events_num":18,"state_list":["clean"],"creation_time_epoch":1660694521,"update_time_epoch":1660694525} diff --git a/tests/fixtures/generic/mdadm-query-raid5-homehost.out b/tests/fixtures/generic/mdadm-query-raid5-homehost.out new file mode 100644 index 000000000..93eb61f7c --- /dev/null +++ b/tests/fixtures/generic/mdadm-query-raid5-homehost.out @@ -0,0 +1,30 @@ +/dev/md0: + Version : 1.2 + Creation Time : Tue Aug 16 17:02:01 2022 + Raid Level : raid5 + Array Size : 405504 (396.00 MiB 415.24 MB) + Used Dev Size : 202752 (198.00 MiB 207.62 MB) + Raid Devices : 3 + Total Devices : 3 + Persistence : Superblock is persistent + + Update Time : Tue Aug 16 17:02:05 2022 + State : clean + Active Devices : 3 + Working Devices : 3 + Failed Devices : 0 + Spare Devices : 0 + + Layout : left-symmetric + Chunk Size : 512K + +Consistency Policy : resync + + Name : foohost:0 + UUID : 07bcc229:6af63526:b1fc14ff:af3524ef + Events : 18 + + Number Major Minor RaidDevice State + 0 254 1 0 active sync /dev/vda1 + 1 254 2 1 active sync /dev/vda2 + 3 254 3 2 active sync /dev/vda3 diff --git a/tests/fixtures/generic/mdadm-query-raid5-meta09.json b/tests/fixtures/generic/mdadm-query-raid5-meta09.json new file mode 100644 index 000000000..32845b684 --- /dev/null +++ b/tests/fixtures/generic/mdadm-query-raid5-meta09.json @@ -0,0 +1 @@ +{"device":"/dev/md0","version":"0.90","creation_time":"Tue Aug 16 17:05:02 2022","raid_level":"raid5","array_size":"408576 (399.00 MiB 418.38 MB)","used_dev_size":"204288 (199.50 MiB 209.19 MB)","raid_devices":3,"total_devices":3,"preferred_minor":0,"persistence":"Superblock is persistent","update_time":"Tue Aug 16 17:05:06 2022","state":"clean","active_devices":3,"working_devices":3,"failed_devices":0,"spare_devices":0,"layout":"left-symmetric","chunk_size":"512K","consistency_policy":"resync","uuid":"4de9c6a6:ce05eada:96fc04a8:108a4af7 (local to host sysrescue)","events":"0.19","device_table":[{"number":0,"major":254,"minor":1,"state":["active","sync"],"device":"/dev/vda1","raid_device":0},{"number":1,"major":254,"minor":2,"state":["active","sync"],"device":"/dev/vda2","raid_device":1},{"number":2,"major":254,"minor":3,"state":["active","sync"],"device":"/dev/vda3","raid_device":2}],"array_size_num":408576,"used_dev_size_num":204288,"uuid_val":"4de9c6a6:ce05eada:96fc04a8:108a4af7","homehost":"sysrescue","chunk_size_num":512,"events_maj":0,"events_min":19,"state_list":["clean"],"creation_time_epoch":1660694702,"update_time_epoch":1660694706} diff --git a/tests/fixtures/generic/mdadm-query-raid5-meta09.out b/tests/fixtures/generic/mdadm-query-raid5-meta09.out new file mode 100644 index 000000000..968de484c --- /dev/null +++ b/tests/fixtures/generic/mdadm-query-raid5-meta09.out @@ -0,0 +1,30 @@ +/dev/md0: + Version : 0.90 + Creation Time : Tue Aug 16 17:05:02 2022 + Raid Level : raid5 + Array Size : 408576 (399.00 MiB 418.38 MB) + Used Dev Size : 204288 (199.50 MiB 209.19 MB) + Raid Devices : 3 + Total Devices : 3 + Preferred Minor : 0 + Persistence : Superblock is persistent + + Update Time : Tue Aug 16 17:05:06 2022 + State : clean + Active Devices : 3 + Working Devices : 3 + Failed Devices : 0 + Spare Devices : 0 + + Layout : left-symmetric + Chunk Size : 512K + +Consistency Policy : resync + + UUID : 4de9c6a6:ce05eada:96fc04a8:108a4af7 (local to host sysrescue) + Events : 0.19 + + Number Major Minor RaidDevice State + 0 254 1 0 active sync /dev/vda1 + 1 254 2 1 active sync /dev/vda2 + 2 254 3 2 active sync /dev/vda3 diff --git a/tests/fixtures/generic/mdadm-query-raid5-ok.json b/tests/fixtures/generic/mdadm-query-raid5-ok.json new file mode 100644 index 000000000..ece182468 --- /dev/null +++ b/tests/fixtures/generic/mdadm-query-raid5-ok.json @@ -0,0 +1 @@ +{"device":"/dev/md0","version":"1.2","creation_time":"Tue Aug 16 16:59:34 2022","raid_level":"raid5","array_size":"405504 (396.00 MiB 415.24 MB)","used_dev_size":"202752 (198.00 MiB 207.62 MB)","raid_devices":3,"total_devices":3,"persistence":"Superblock is persistent","update_time":"Tue Aug 16 16:59:39 2022","state":"clean","active_devices":3,"working_devices":3,"failed_devices":0,"spare_devices":0,"layout":"left-symmetric","chunk_size":"512K","consistency_policy":"resync","name":"sysrescue:0 (local to host sysrescue)","uuid":"4d798d62:adfb6a9f:cbe9a393:b79cd3f4","events":"18","device_table":[{"number":0,"major":254,"minor":1,"state":["active","sync"],"device":"/dev/vda1","raid_device":0},{"number":1,"major":254,"minor":2,"state":["active","sync"],"device":"/dev/vda2","raid_device":1},{"number":3,"major":254,"minor":3,"state":["active","sync"],"device":"/dev/vda3","raid_device":2}],"array_size_num":405504,"used_dev_size_num":202752,"name_val":"sysrescue:0","homehost":"sysrescue","uuid_val":"4d798d62:adfb6a9f:cbe9a393:b79cd3f4","chunk_size_num":512,"events_num":18,"state_list":["clean"],"creation_time_epoch":1660694374,"update_time_epoch":1660694379} diff --git a/tests/fixtures/generic/mdadm-query-raid5-ok.out b/tests/fixtures/generic/mdadm-query-raid5-ok.out new file mode 100644 index 000000000..524a31952 --- /dev/null +++ b/tests/fixtures/generic/mdadm-query-raid5-ok.out @@ -0,0 +1,30 @@ +/dev/md0: + Version : 1.2 + Creation Time : Tue Aug 16 16:59:34 2022 + Raid Level : raid5 + Array Size : 405504 (396.00 MiB 415.24 MB) + Used Dev Size : 202752 (198.00 MiB 207.62 MB) + Raid Devices : 3 + Total Devices : 3 + Persistence : Superblock is persistent + + Update Time : Tue Aug 16 16:59:39 2022 + State : clean + Active Devices : 3 + Working Devices : 3 + Failed Devices : 0 + Spare Devices : 0 + + Layout : left-symmetric + Chunk Size : 512K + +Consistency Policy : resync + + Name : sysrescue:0 (local to host sysrescue) + UUID : 4d798d62:adfb6a9f:cbe9a393:b79cd3f4 + Events : 18 + + Number Major Minor RaidDevice State + 0 254 1 0 active sync /dev/vda1 + 1 254 2 1 active sync /dev/vda2 + 3 254 3 2 active sync /dev/vda3 diff --git a/tests/test_mdadm.py b/tests/test_mdadm.py index dd7e68413..d095fc89f 100644 --- a/tests/test_mdadm.py +++ b/tests/test_mdadm.py @@ -91,6 +91,45 @@ def setUp(self): with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-query-raid1-syncing.out'), 'r', encoding='utf-8') as f: self.mdadm_query_raid1_syncing = f.read() + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-examine-container1.out'), 'r', encoding='utf-8') as f: + self.mdadm_examine_container1 = f.read() + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-examine-container2-dev1.out'), 'r', encoding='utf-8') as f: + self.mdadm_examine_container2_dev1 = f.read() + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-examine-container2-dev2.out'), 'r', encoding='utf-8') as f: + self.mdadm_examine_container2_dev2 = f.read() + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-examine-raid5-homehost.out'), 'r', encoding='utf-8') as f: + self.mdadm_examine_raid5_homehost = f.read() + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-examine-raid5-meta09.out'), 'r', encoding='utf-8') as f: + self.mdadm_examine_raid5_meta09 = f.read() + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-examine-raid5-ok.out'), 'r', encoding='utf-8') as f: + self.mdadm_examine_raid5_ok = f.read() + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-query-container1-member.out'), 'r', encoding='utf-8') as f: + self.mdadm_query_container1_member = f.read() + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-query-container1-root.out'), 'r', encoding='utf-8') as f: + self.mdadm_query_container1_root = f.read() + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-query-container2-member.out'), 'r', encoding='utf-8') as f: + self.mdadm_query_container2_member = f.read() + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-query-container2-root.out'), 'r', encoding='utf-8') as f: + self.mdadm_query_container2_root = f.read() + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-query-raid5-homehost.out'), 'r', encoding='utf-8') as f: + self.mdadm_query_raid5_homehost = f.read() + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-query-raid5-meta09.out'), 'r', encoding='utf-8') as f: + self.mdadm_query_raid5_meta09 = f.read() + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-query-raid5-ok.out'), 'r', encoding='utf-8') as f: + self.mdadm_query_raid5_ok = f.read() + # output with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-examine-raid0-offline.json'), 'r', encoding='utf-8') as f: self.mdadm_examine_raid0_offline_json = json.loads(f.read()) @@ -173,6 +212,45 @@ def setUp(self): with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-query-raid1-syncing.json'), 'r', encoding='utf-8') as f: self.mdadm_query_raid1_syncing_json = json.loads(f.read()) + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-examine-container1.json'), 'r', encoding='utf-8') as f: + self.mdadm_examine_container1_json = json.loads(f.read()) + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-examine-container2-dev1.json'), 'r', encoding='utf-8') as f: + self.mdadm_examine_container2_dev1_json = json.loads(f.read()) + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-examine-container2-dev2.json'), 'r', encoding='utf-8') as f: + self.mdadm_examine_container2_dev2_json = json.loads(f.read()) + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-examine-raid5-homehost.json'), 'r', encoding='utf-8') as f: + self.mdadm_examine_raid5_homehost_json = json.loads(f.read()) + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-examine-raid5-meta09.json'), 'r', encoding='utf-8') as f: + self.mdadm_examine_raid5_meta09_json = json.loads(f.read()) + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-examine-raid5-ok.json'), 'r', encoding='utf-8') as f: + self.mdadm_examine_raid5_ok_json = json.loads(f.read()) + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-query-container1-member.json'), 'r', encoding='utf-8') as f: + self.mdadm_query_container1_member_json = json.loads(f.read()) + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-query-container1-root.json'), 'r', encoding='utf-8') as f: + self.mdadm_query_container1_root_json = json.loads(f.read()) + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-query-container2-member.json'), 'r', encoding='utf-8') as f: + self.mdadm_query_container2_member_json = json.loads(f.read()) + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-query-container2-root.json'), 'r', encoding='utf-8') as f: + self.mdadm_query_container2_root_json = json.loads(f.read()) + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-query-raid5-homehost.json'), 'r', encoding='utf-8') as f: + self.mdadm_query_raid5_homehost_json = json.loads(f.read()) + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-query-raid5-meta09.json'), 'r', encoding='utf-8') as f: + self.mdadm_query_raid5_meta09_json = json.loads(f.read()) + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/mdadm-query-raid5-ok.json'), 'r', encoding='utf-8') as f: + self.mdadm_query_raid5_ok_json = json.loads(f.read()) + def test_mdadm_nodata(self): """ @@ -370,5 +448,96 @@ def test_mdadm_query_raid1_syncing(self): self.assertEqual(jc.parsers.mdadm.parse(self.mdadm_query_raid1_syncing, quiet=True), self.mdadm_query_raid1_syncing_json) + def test_mdadm_examine_container1(self): + """ + Test 'mdadm --examine' on container 1 + """ + self.assertEqual(jc.parsers.mdadm.parse(self.mdadm_examine_container1, quiet=True), self.mdadm_examine_container1_json) + + + def test_mdadm_examine_container2_dev1(self): + """ + Test 'mdadm --examine' on container 1 dev 1 + """ + self.assertEqual(jc.parsers.mdadm.parse(self.mdadm_examine_container2_dev1, quiet=True), self.mdadm_examine_container2_dev1_json) + + + def test_mdadm_examine_container2_dev2(self): + """ + Test 'mdadm --examine' on container 1 dev 2 + """ + self.assertEqual(jc.parsers.mdadm.parse(self.mdadm_examine_container2_dev2, quiet=True), self.mdadm_examine_container2_dev2_json) + + + def test_mdadm_examine_raid5_homehost(self): + """ + Test 'mdadm --examine' on RAID5 homehost + """ + self.assertEqual(jc.parsers.mdadm.parse(self.mdadm_examine_raid5_homehost, quiet=True), self.mdadm_examine_raid5_homehost_json) + + + def test_mdadm_examine_raid5_meta09(self): + """ + Test 'mdadm --examine' on RAID5 on v0.9 + """ + self.assertEqual(jc.parsers.mdadm.parse(self.mdadm_examine_raid5_meta09, quiet=True), self.mdadm_examine_raid5_meta09_json) + + + def test_mdadm_examine_raid5_ok(self): + """ + Test 'mdadm --examine' on ok RAID5 + """ + self.assertEqual(jc.parsers.mdadm.parse(self.mdadm_examine_raid5_ok, quiet=True), self.mdadm_examine_raid5_ok_json) + + + def test_mdadm_query_container1_member(self): + """ + Test 'mdadm --query' container1 member + """ + self.assertEqual(jc.parsers.mdadm.parse(self.mdadm_query_container1_member, quiet=True), self.mdadm_query_container1_member_json) + + + def test_mdadm_query_container1_root(self): + """ + Test 'mdadm --query' container1 root + """ + self.assertEqual(jc.parsers.mdadm.parse(self.mdadm_query_container1_root, quiet=True), self.mdadm_query_container1_root_json) + + + def test_mdadm_query_container2_member(self): + """ + Test 'mdadm --query' container2 member + """ + self.assertEqual(jc.parsers.mdadm.parse(self.mdadm_query_container2_member, quiet=True), self.mdadm_query_container2_member_json) + + + def test_mdadm_query_container2_root(self): + """ + Test 'mdadm --query' container2 root + """ + self.assertEqual(jc.parsers.mdadm.parse(self.mdadm_query_container2_root, quiet=True), self.mdadm_query_container2_root_json) + + + def test_mdadm_query_raid5_homehost(self): + """ + Test 'mdadm --query' RAID5 with homehost + """ + self.assertEqual(jc.parsers.mdadm.parse(self.mdadm_query_raid5_homehost, quiet=True), self.mdadm_query_raid5_homehost_json) + + + def test_mdadm_query_raid5_meta09(self): + """ + Test 'mdadm --query' RAID5 on v0.9 + """ + self.assertEqual(jc.parsers.mdadm.parse(self.mdadm_query_raid5_meta09, quiet=True), self.mdadm_query_raid5_meta09_json) + + + def test_mdadm_query_raid5_ok(self): + """ + Test 'mdadm --query' on ok RAID5 + """ + self.assertEqual(jc.parsers.mdadm.parse(self.mdadm_query_raid5_ok, quiet=True), self.mdadm_query_raid5_ok_json) + + if __name__ == '__main__': unittest.main() From 7fbfec4e5af61b5d5ff407b9f4c18577fb6355a0 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sat, 20 Aug 2022 11:13:32 -0700 Subject: [PATCH 104/116] remove unneeded raw strings --- jc/parsers/syslog.py | 4 ++-- jc/parsers/syslog_s.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/jc/parsers/syslog.py b/jc/parsers/syslog.py index afaf6f0d5..38b924c24 100644 --- a/jc/parsers/syslog.py +++ b/jc/parsers/syslog.py @@ -121,8 +121,8 @@ class info(): # https://www.rfc-editor.org/rfc/rfc5424.html#section-6 escape_map = { r'\\': '\\', - r'\"': r'"', - r'\]': r']' + r'\"': '"', + r'\]': ']' } diff --git a/jc/parsers/syslog_s.py b/jc/parsers/syslog_s.py index 896af5629..3524f7b94 100644 --- a/jc/parsers/syslog_s.py +++ b/jc/parsers/syslog_s.py @@ -99,8 +99,8 @@ class info(): # https://www.rfc-editor.org/rfc/rfc5424.html#section-6 escape_map = { r'\\': '\\', - r'\"': r'"', - r'\]': r']' + r'\"': '"', + r'\]': ']' } From 935e114d404444a30c5869150864babe642757a5 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sat, 20 Aug 2022 11:25:56 -0700 Subject: [PATCH 105/116] add syslog streaming tests --- .../generic/syslog-3164-streaming.json | 1 + .../generic/syslog-5424-streaming.json | 1 + tests/test_syslog_bsd_s.py | 37 +++++++++++++++++++ tests/test_syslog_s.py | 37 +++++++++++++++++++ 4 files changed, 76 insertions(+) create mode 100644 tests/fixtures/generic/syslog-3164-streaming.json create mode 100644 tests/fixtures/generic/syslog-5424-streaming.json create mode 100644 tests/test_syslog_bsd_s.py create mode 100644 tests/test_syslog_s.py diff --git a/tests/fixtures/generic/syslog-3164-streaming.json b/tests/fixtures/generic/syslog-3164-streaming.json new file mode 100644 index 000000000..093a62ee7 --- /dev/null +++ b/tests/fixtures/generic/syslog-3164-streaming.json @@ -0,0 +1 @@ +[{"priority":34,"date":"Oct 11 22:14:15","hostname":"mymachine","tag":"su","content":"'su root' failed for lonvick on /dev/pts/8"},{"unparsable":"<3444>Oct 11 22:14:15 mymachine su: 'su root' failed for lonvick on /dev/pts/8"},{"priority":null,"date":"Oct 11 22:14:15","hostname":"mymachine","tag":"su","content":"'su root' failed for lonvick on /dev/pts/8"},{"priority":35,"date":"Oct 12 22:14:15","hostname":"client_machine","tag":"su","content":"'su root' failed for joe on /dev/pts/2"},{"priority":35,"date":"Mar 7 04:02:16","hostname":"avas","tag":"clamd","content":"[11165]: /var/amavis/amavis-20040307T033734-10329/parts/part-00003: Worm.Mydoom.F FOUND"},{"priority":null,"date":"Mar 7 04:05:55","hostname":"avas","tag":"clamd","content":"[11240]: /var/amavis/amavis-20040307T035901-10615/parts/part-00002: Worm.SomeFool.Gen-1 FOUND"},{"priority":5,"date":"Mar 7 09:00:51","hostname":"avas","tag":"clamd","content":"[27173]: SelfCheck: Database status OK."},{"priority":null,"date":"Mar 7 05:59:02","hostname":"avas","tag":"clamd","content":"[27173]: Database correctly reloaded (20400 viruses)"},{"priority":null,"date":"Mar 7 04:02:16","hostname":"avas","tag":"clamd","content":"[11165]: /var/amavis/amavis-20040307T033734-10329/parts/part-00003: Worm.Mydoom.F FOUND"},{"priority":null,"date":"Mar 7 04:05:55","hostname":"avas","tag":"clamd","content":"[11240]: /var/amavis/amavis-20040307T035901-10615/parts/part-00002: Worm.SomeFool.Gen-1 FOUND"},{"priority":null,"date":"Mar 7 09:00:51","hostname":"avas","tag":"clamd","content":"[27173]: SelfCheck: Database status OK."},{"priority":null,"date":"Mar 7 05:59:02","hostname":"avas","tag":"clamd","content":"[27173]: Database correctly reloaded (20400 viruses)"},{"priority":null,"date":"Mar 7 11:14:35","hostname":"avas","tag":"dccd","content":"[13284]: 21 requests/sec are too many from anonymous 205.201.1.56,2246"},{"priority":null,"date":"Mar 8 00:22:57","hostname":"avas","tag":"dccifd","content":"[9933]: write(MTA socket,4): Broken pipe"},{"priority":null,"date":"Mar 7 21:23:22","hostname":"avas","tag":"dccifd","content":"[6191]: missing message body"},{"priority":null,"date":"Mar 9 16:05:17","hostname":"avas","tag":"named","content":"[12045]: zone PLNet/IN: refresh: non-authoritative answer from master 10.0.0.253#53"},{"priority":null,"date":"Mar 10 00:38:16","hostname":"avas","tag":"dccifd","content":"[23069]: continue not asking DCC 17 seconds after failure"},{"priority":null,"date":"Mar 10 09:42:11","hostname":"avas","tag":"named","content":"client 127.0.0.1#55524: query: 23.68.27.142.sa-trusted.bondedsender.org IN TXT"},{"priority":null,"date":"Mar 9 03:48:07","hostname":"avas","tag":"dccd","content":"[145]: automatic dbclean; starting `dbclean -DPq -i 1189 -L info,local5.notice -L error,local5.err`"},{"priority":null,"date":"Mar 9 11:58:18","hostname":"avas","tag":"kernel","content":"i810_audio: Connection 0 with codec id 2"},{"priority":null,"date":"Mar 9 19:41:13","hostname":"avas","tag":"dccd","content":"[3004]: \"packet length 44 too small for REPORT\" sent to client 1 at 194.63.250.215,47577"},{"priority":null,"date":"Mar 8 09:01:07","hostname":"avas","tag":"sshd","content":"(pam_unix)[21839]: session opened for user tom by (uid=35567)"},{"priority":null,"date":"Mar 8 03:52:04","hostname":"avas","tag":"dccd","content":"[13284]: 1.2.32 database /home/dcc/dcc_db reopened with 997 MByte window"},{"priority":null,"date":"Mar 8 16:05:26","hostname":"avas","tag":"arpwatch","content":"listening on eth0"},{"priority":null,"date":"Mar 10 10:00:06","hostname":"avas","tag":"named","content":"[6986]: zone PLNet/IN: refresh: non-authoritative answer from master 192.75.26.21#53"},{"priority":null,"date":"Mar 10 10:00:10","hostname":"avas","tag":"named","content":"[6986]: client 127.0.0.1#55867: query: mail.canfor.ca IN MX"},{"priority":null,"date":"Mar 8 15:18:40","hostname":"avas","tag":null,"content":"last message repeated 11 times"},{"priority":null,"date":"Mar 8 15:18:40","hostname":"127:0:ab::1","tag":"sshd","content":"unauthorized request"},{"priority":null,"date":"Mar 8 15:18:40","hostname":"server.example.com","tag":"sshd","content":"unauthorized request"},{"priority":null,"date":"Mar 8 15:18:40","hostname":"192.168.1.1","tag":"sshd","content":"unauthorized request"},{"priority":35,"date":"Mar 8 15:18:40","hostname":"server.example.com","tag":"sshd","content":"unauthorized request"},{"unparsable":"<7>unparsable line"}] diff --git a/tests/fixtures/generic/syslog-5424-streaming.json b/tests/fixtures/generic/syslog-5424-streaming.json new file mode 100644 index 000000000..42528945c --- /dev/null +++ b/tests/fixtures/generic/syslog-5424-streaming.json @@ -0,0 +1 @@ +[{"priority":165,"version":1,"timestamp":"2003-08-24T05:14:15.000003-07:00","hostname":"192.0.2.1","appname":"myproc","proc_id":8710,"msg_id":null,"structured_data":null,"message":"%% It's time to make the do-nuts.","timestamp_epoch":1061727255,"timestamp_epoch_utc":null},{"unparsable":"<34>1 203-10-11T22:14:15.003Z mymachine.example.com su - ID47 - BOM'su root' failed for lonvick on /dev/pts/8"},{"priority":34,"version":1,"timestamp":"2003-10-11T22:14:15.003Z","hostname":"mymachine.example.com","appname":"su","proc_id":null,"msg_id":"ID47","structured_data":null,"message":"BOM'su root' failed for lonvick on /dev/pts/8","timestamp_epoch":1065935655,"timestamp_epoch_utc":1065910455},{"priority":165,"version":1,"timestamp":"2003-08-24T05:14:15.000003-07:00","hostname":"192.0.2.1","appname":"myproc","proc_id":8710,"msg_id":null,"structured_data":null,"message":"%% It's time to make the do-nuts.","timestamp_epoch":1061727255,"timestamp_epoch_utc":null},{"priority":null,"version":1,"timestamp":"2003-10-11T22:14:15.003Z","hostname":"mymachine.example.com","appname":"su","proc_id":null,"msg_id":"ID47","structured_data":null,"message":"BOM'su root' failed for lonvick on /dev/pts/8","timestamp_epoch":1065935655,"timestamp_epoch_utc":1065910455},{"priority":190,"version":1,"timestamp":"2003-10-11T22:14:15.003Z","hostname":"mymachine.example.com","appname":"evntslog","proc_id":null,"msg_id":"ID47","structured_data":[{"identity":"exampleSDID@32473","parameters":{"iut":"3","eventSource":"Application","eventID":"10] 11"}}],"message":"BOMAn application event log entry..[ ] sadasd","timestamp_epoch":1065935655,"timestamp_epoch_utc":1065910455},{"unparsable":"<190)1 2003-10-11T22:14:15.003Z mymachine.example.com evntslog - ID47 [exampleSDID@32473 iut=\"3\" eventSource=\"Application\" eventID=\"10\\] 11\"] BOMAn application event log entry..[ ] sadasd"},{"priority":null,"version":null,"timestamp":"2003-10-11T22:14:15.003Z","hostname":"mymachine.example.com","appname":"evntslog","proc_id":null,"msg_id":"ID47","structured_data":[{"identity":"exampleSDID@32473","parameters":{"iut":"3","eventSource":"Application","eventID":"1011"}},{"identity":"examplePriority@32473","parameters":{"class":"high"}}],"message":null,"timestamp_epoch":1065935655,"timestamp_epoch_utc":1065910455},{"priority":1,"version":12,"timestamp":null,"hostname":"mymachine","appname":null,"proc_id":null,"msg_id":"ID47","structured_data":null,"message":"asd asdaasd"}] diff --git a/tests/test_syslog_bsd_s.py b/tests/test_syslog_bsd_s.py new file mode 100644 index 000000000..82c24b0c2 --- /dev/null +++ b/tests/test_syslog_bsd_s.py @@ -0,0 +1,37 @@ +import os +import json +import unittest +import jc.parsers.syslog_bsd_s + +THIS_DIR = os.path.dirname(os.path.abspath(__file__)) + +# To create streaming output use: +# $ cat syslog.out | jc --syslog-bsd-s | jello -c > syslog-bsd-streaming.json + + +class MyTests(unittest.TestCase): + + def setUp(self): + # input + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/syslog-3164.out'), 'r', encoding='utf-8') as f: + self.syslog_bsd = f.read() + + # output + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/syslog-3164-streaming.json'), 'r', encoding='utf-8') as f: + self.syslog_bsd_streaming_json = json.loads(f.read()) + + def test_syslog_bsd_s_nodata(self): + """ + Test 'syslog_bsd' with no data + """ + self.assertEqual(list(jc.parsers.syslog_bsd_s.parse([], quiet=True)), []) + + def test_syslog_bsd_s(self): + """ + Test bsd Syslog + """ + self.assertEqual(list(jc.parsers.syslog_bsd_s.parse(self.syslog_bsd.splitlines(), quiet=True)), self.syslog_bsd_streaming_json) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_syslog_s.py b/tests/test_syslog_s.py new file mode 100644 index 000000000..97c5adff0 --- /dev/null +++ b/tests/test_syslog_s.py @@ -0,0 +1,37 @@ +import os +import json +import unittest +import jc.parsers.syslog_s + +THIS_DIR = os.path.dirname(os.path.abspath(__file__)) + +# To create streaming output use: +# $ cat syslog.out | jc --syslog-s | jello -c > syslog-streaming.json + + +class MyTests(unittest.TestCase): + + def setUp(self): + # input + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/syslog-5424.out'), 'r', encoding='utf-8') as f: + self.syslog = f.read() + + # output + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/syslog-5424-streaming.json'), 'r', encoding='utf-8') as f: + self.syslog_streaming_json = json.loads(f.read()) + + def test_syslog_s_nodata(self): + """ + Test 'syslog' with no data + """ + self.assertEqual(list(jc.parsers.syslog_s.parse([], quiet=True)), []) + + def test_syslog_s(self): + """ + Test syslog file + """ + self.assertEqual(list(jc.parsers.syslog_s.parse(self.syslog.splitlines(), quiet=True)), self.syslog_streaming_json) + + +if __name__ == '__main__': + unittest.main() From 944ae9f8d9243b9b0aea4cd447ea8da02cce05af Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sat, 20 Aug 2022 11:31:52 -0700 Subject: [PATCH 106/116] doc update --- CHANGELOG | 3 ++- docs/parsers/cef.md | 2 +- docs/parsers/traceroute.md | 2 +- jc/parsers/cef.py | 2 +- man/jc.1 | 2 +- 5 files changed, 6 insertions(+), 5 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 523c1462c..6316bb44a 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -5,7 +5,8 @@ xxxxxxxx v1.21.0 - Add Syslog string parsers (RFC 3164 and RFC 5424) - Add CEF string parser - Add PLIST file parser (XML and binary support) -- Add `mdadm` command parser tested on linux (IN PROGRESS) +- Add `-n` support to the `traceroute` parser +- Add `mdadm` command parser tested on linux - Add `--time-out` or `-t` option to add a UTC timestamp to the JSON output - Fix `lsusb` command parser for output containing a `Device Qualifier` and `Binary Object Store Descriptor` sections diff --git a/docs/parsers/cef.md b/docs/parsers/cef.md index cb2753151..a75b3db6b 100644 --- a/docs/parsers/cef.md +++ b/docs/parsers/cef.md @@ -17,7 +17,7 @@ and the values are converted to their respective types. Extra naive and UTC epoch timestamps are added where appropriate per the CEF specification. To preserve escaping and original keynames and to prevent type conversions -use the `--raw` or `raw=True` option in the `parse()` function. +use the `--raw` CLI option or `raw=True` param in the `parse()` function. Usage (cli): diff --git a/docs/parsers/traceroute.md b/docs/parsers/traceroute.md index 1a0f73472..eb7b5c8be 100644 --- a/docs/parsers/traceroute.md +++ b/docs/parsers/traceroute.md @@ -143,4 +143,4 @@ Returns: ### Parser Information Compatibility: linux, darwin, freebsd -Version 1.5 by Kelly Brazil (kellyjonbrazil@gmail.com) +Version 1.6 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/jc/parsers/cef.py b/jc/parsers/cef.py index f5071b4f5..de7c3f534 100644 --- a/jc/parsers/cef.py +++ b/jc/parsers/cef.py @@ -12,7 +12,7 @@ UTC epoch timestamps are added where appropriate per the CEF specification. To preserve escaping and original keynames and to prevent type conversions -use the `--raw` or `raw=True` option in the `parse()` function. +use the `--raw` CLI option or `raw=True` param in the `parse()` function. Usage (cli): diff --git a/man/jc.1 b/man/jc.1 index 4314bdce0..f101dc077 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -1,4 +1,4 @@ -.TH jc 1 2022-08-19 1.21.0 "JSON Convert" +.TH jc 1 2022-08-20 1.21.0 "JSON Convert" .SH NAME \fBjc\fP \- JSON Convert JSONifies the output of many CLI tools and file-types .SH SYNOPSIS From b6cd123719c42e8e902cf4f2f62ebfd8c4bd5cc3 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sat, 20 Aug 2022 12:30:52 -0700 Subject: [PATCH 107/116] doc update --- docs/parsers/cef.md | 15 ++++++++++---- docs/parsers/syslog_bsd_s.md | 2 ++ docs/parsers/syslog_s.md | 2 ++ jc/parsers/cef.py | 40 +++++++++++++++++++++++++++--------- jc/parsers/syslog_bsd_s.py | 2 ++ jc/parsers/syslog_s.py | 2 ++ 6 files changed, 49 insertions(+), 14 deletions(-) diff --git a/docs/parsers/cef.md b/docs/parsers/cef.md index a75b3db6b..15f602d82 100644 --- a/docs/parsers/cef.md +++ b/docs/parsers/cef.md @@ -16,6 +16,9 @@ Extended fields, as defined in the CEF specification, are relabeled and the values are converted to their respective types. Extra naive and UTC epoch timestamps are added where appropriate per the CEF specification. +A warning message to `STDERR` will be printed if an unparsable line is found +unless `--quiet` or `quiet=True` is used. + To preserve escaping and original keynames and to prevent type conversions use the `--raw` CLI option or `raw=True` param in the `parse()` function. @@ -40,15 +43,17 @@ See: https://www.microfocus.com/documentation/arcsight/arcsight-smartconnectors- "deviceProduct": string, "deviceVersion": string, "deviceEventClassId": string, + "deviceEventClassIdNum": integer/null, "name": string, "agentSeverity": string/integer, "agentSeverityString": string, - "agentSeverityNum": integer, + "agentSeverityNum": integer/null, "CEFVersion": integer, string/integer/float, # [0] - "_epoch": integer, # [1] - "_epoch_utc": integer, # [2] - string + "_epoch": integer/null, # [1] + "_epoch_utc": integer/null, # [2] + string, + "unparsable": string # [3] } ] @@ -59,6 +64,8 @@ See: https://www.microfocus.com/documentation/arcsight/arcsight-smartconnectors- [2] Timezone-aware calculated epoch timestamp. (UTC only) This value will be null if a UTC timezone cannot be extracted from the original timestamp string value. + [3] this field exists if the CEF line is not parsable. The value + is the original syslog line. Examples: diff --git a/docs/parsers/syslog_bsd_s.md b/docs/parsers/syslog_bsd_s.md index 90217cbd4..d7b141c3c 100644 --- a/docs/parsers/syslog_bsd_s.md +++ b/docs/parsers/syslog_bsd_s.md @@ -49,10 +49,12 @@ Examples: $ cat syslog.txt | jc --syslog-bsd-s -p {"priority":34,"date":"Oct 11 22:14:15","hostname":"mymachine","t...} + {"priority":34,"date":"Oct 11 22:14:16","hostname":"mymachine","t...} ... $ cat syslog.txt | jc --syslog-bsd-s -p -r {"priority":"34","date":"Oct 11 22:14:15","hostname":"mymachine","...} + {"priority":"34","date":"Oct 11 22:14:16","hostname":"mymachine","...} ... diff --git a/docs/parsers/syslog_s.md b/docs/parsers/syslog_s.md index f2604845b..d15b05f75 100644 --- a/docs/parsers/syslog_s.md +++ b/docs/parsers/syslog_s.md @@ -72,10 +72,12 @@ Examples: $ cat syslog.txt | jc --syslog-s -p {"priority":165,"version":1,"timestamp":"2003-08-24T05:14:15.000003-...} + {"priority":165,"version":1,"timestamp":"2003-08-24T05:14:16.000003-...} ... $ cat syslog.txt | jc --syslog-s -p -r {"priority":"165","version":"1","timestamp":"2003-08-24T05:14:15.000...} + {"priority":"165","version":"1","timestamp":"2003-08-24T05:15:15.000...} ... diff --git a/jc/parsers/cef.py b/jc/parsers/cef.py index de7c3f534..ceedf0ab6 100644 --- a/jc/parsers/cef.py +++ b/jc/parsers/cef.py @@ -11,6 +11,9 @@ and the values are converted to their respective types. Extra naive and UTC epoch timestamps are added where appropriate per the CEF specification. +A warning message to `STDERR` will be printed if an unparsable line is found +unless `--quiet` or `quiet=True` is used. + To preserve escaping and original keynames and to prevent type conversions use the `--raw` CLI option or `raw=True` param in the `parse()` function. @@ -35,15 +38,17 @@ "deviceProduct": string, "deviceVersion": string, "deviceEventClassId": string, + "deviceEventClassIdNum": integer/null, "name": string, "agentSeverity": string/integer, "agentSeverityString": string, - "agentSeverityNum": integer, + "agentSeverityNum": integer/null, "CEFVersion": integer, string/integer/float, # [0] - "_epoch": integer, # [1] - "_epoch_utc": integer, # [2] - string + "_epoch": integer/null, # [1] + "_epoch_utc": integer/null, # [2] + string, + "unparsable": string # [3] } ] @@ -54,6 +59,8 @@ [2] Timezone-aware calculated epoch timestamp. (UTC only) This value will be null if a UTC timezone cannot be extracted from the original timestamp string value. + [3] this field exists if the CEF line is not parsable. The value + is the original syslog line. Examples: @@ -357,15 +364,21 @@ def _process(proc_data: List[Dict]) -> List[Dict]: if key in int_list: item[key] = jc.utils.convert_to_int(item[key]) - # set SeverityString and SeverityNum: + # set agentSeverityString and agentSeverityNum: if 'agentSeverity' in item: - if isinstance(item['agentSeverity'], str) and item['agentSeverity'].lower() in severity_set: + if item['agentSeverity'].lower() in severity_set: item['agentSeverityString'] = item['agentSeverity'] item['agentSeverityNum'] = None else: - item['agentSeverity'] = int(item['agentSeverity']) - item['agentSeverityString'] = severity_map[item['agentSeverity']] - item['agentSeverityNum'] = item['agentSeverity'] + try: + item['agentSeverityString'] = severity_map[int(item['agentSeverity'])] + item['agentSeverityNum'] = int(item['agentSeverity']) + except Exception: + pass + + # set deviceEventClassIdNum: + if 'deviceEventClassId' in item: + item['deviceEventClassIdNum'] = jc.utils.convert_to_int(item['deviceEventClassId']) return proc_data @@ -395,6 +408,13 @@ def parse( if jc.utils.has_data(data): for line in filter(None, data.splitlines()): - raw_output.append(_pycef_parse(line)) + try: + raw_output.append(_pycef_parse(line)) + except Exception: + if not quiet: + jc.utils.warning_message( + [f'Unparsable CEF line found: {line}'] + ) + raw_output.append({"unparsable": line}) return raw_output if raw else _process(raw_output) diff --git a/jc/parsers/syslog_bsd_s.py b/jc/parsers/syslog_bsd_s.py index c50e2f980..838953c0d 100644 --- a/jc/parsers/syslog_bsd_s.py +++ b/jc/parsers/syslog_bsd_s.py @@ -44,10 +44,12 @@ $ cat syslog.txt | jc --syslog-bsd-s -p {"priority":34,"date":"Oct 11 22:14:15","hostname":"mymachine","t...} + {"priority":34,"date":"Oct 11 22:14:16","hostname":"mymachine","t...} ... $ cat syslog.txt | jc --syslog-bsd-s -p -r {"priority":"34","date":"Oct 11 22:14:15","hostname":"mymachine","...} + {"priority":"34","date":"Oct 11 22:14:16","hostname":"mymachine","...} ... """ from typing import Dict, Iterable, Union diff --git a/jc/parsers/syslog_s.py b/jc/parsers/syslog_s.py index 3524f7b94..aa790b030 100644 --- a/jc/parsers/syslog_s.py +++ b/jc/parsers/syslog_s.py @@ -67,10 +67,12 @@ $ cat syslog.txt | jc --syslog-s -p {"priority":165,"version":1,"timestamp":"2003-08-24T05:14:15.000003-...} + {"priority":165,"version":1,"timestamp":"2003-08-24T05:14:16.000003-...} ... $ cat syslog.txt | jc --syslog-s -p -r {"priority":"165","version":"1","timestamp":"2003-08-24T05:14:15.000...} + {"priority":"165","version":"1","timestamp":"2003-08-24T05:15:15.000...} ... """ from typing import List, Dict, Iterable, Union, Optional From 7ad0f8a17f31376ec3765cc305cb00a2198808a6 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sat, 20 Aug 2022 12:37:11 -0700 Subject: [PATCH 108/116] formatting --- jc/parsers/cef.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jc/parsers/cef.py b/jc/parsers/cef.py index ceedf0ab6..300bf0ace 100644 --- a/jc/parsers/cef.py +++ b/jc/parsers/cef.py @@ -59,7 +59,7 @@ [2] Timezone-aware calculated epoch timestamp. (UTC only) This value will be null if a UTC timezone cannot be extracted from the original timestamp string value. - [3] this field exists if the CEF line is not parsable. The value + [3] This field exists if the CEF line is not parsable. The value is the original syslog line. Examples: @@ -85,6 +85,7 @@ "myDate_epoch": 1667939400, "myDate_epoch_utc": null, "myFloat": 3.14, + "deviceEventClassIdNum": 4000000 "agentSeverityString": "Medium", "agentSeverityNum": 6 } From 7c3584500695cfb60cd4b971f520fb88792210d4 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sat, 20 Aug 2022 12:37:45 -0700 Subject: [PATCH 109/116] formatting --- docs/parsers/cef.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/parsers/cef.md b/docs/parsers/cef.md index 15f602d82..04e082ae0 100644 --- a/docs/parsers/cef.md +++ b/docs/parsers/cef.md @@ -64,7 +64,7 @@ See: https://www.microfocus.com/documentation/arcsight/arcsight-smartconnectors- [2] Timezone-aware calculated epoch timestamp. (UTC only) This value will be null if a UTC timezone cannot be extracted from the original timestamp string value. - [3] this field exists if the CEF line is not parsable. The value + [3] This field exists if the CEF line is not parsable. The value is the original syslog line. Examples: @@ -90,6 +90,7 @@ Examples: "myDate_epoch": 1667939400, "myDate_epoch_utc": null, "myFloat": 3.14, + "deviceEventClassIdNum": 4000000 "agentSeverityString": "Medium", "agentSeverityNum": 6 } From 23e81bc3fe129808f12ca9e6df4f178ba38f7cfe Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sat, 20 Aug 2022 13:30:39 -0700 Subject: [PATCH 110/116] add cef streaming parser. doc updates --- README.md | 1 + completions/jc_bash_completion.sh | 2 +- completions/jc_zsh_completion.sh | 3 +- docs/parsers/cef.md | 4 +- docs/parsers/cef_s.md | 124 ++++++++++++ jc/lib.py | 1 + jc/parsers/cef.py | 4 +- jc/parsers/cef_s.py | 320 ++++++++++++++++++++++++++++++ man/jc.1 | 5 + 9 files changed, 458 insertions(+), 6 deletions(-) create mode 100644 docs/parsers/cef_s.md create mode 100644 jc/parsers/cef_s.py diff --git a/README.md b/README.md index 38e652902..e4b0f32dd 100644 --- a/README.md +++ b/README.md @@ -158,6 +158,7 @@ option. | ` --asciitable-m` | multi-line ASCII and Unicode table parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/asciitable_m) | | ` --blkid` | `blkid` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/blkid) | | ` --cef` | CEF string parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/cef) | +| ` --cef-s` | CEF string streaming parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/cef_s) | | ` --chage` | `chage --list` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/chage) | | ` --cksum` | `cksum` and `sum` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/cksum) | | ` --crontab` | `crontab` command and file parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/crontab) | diff --git a/completions/jc_bash_completion.sh b/completions/jc_bash_completion.sh index 4431b3f85..8b2901d6b 100644 --- a/completions/jc_bash_completion.sh +++ b/completions/jc_bash_completion.sh @@ -4,7 +4,7 @@ _jc() jc_about_options jc_about_mod_options jc_help_options jc_special_options jc_commands=(acpi airport arp blkid chage cksum crontab date df dig dmidecode dpkg du env file finger free git gpg hciconfig id ifconfig iostat iptables iw jobs last lastb ls lsblk lsmod lsof lsusb md5 md5sum mdadm mount mpstat netstat nmcli ntpq pidstat ping ping6 pip pip3 postconf printenv ps route rpm rsync sfdisk sha1sum sha224sum sha256sum sha384sum sha512sum shasum ss stat sum sysctl systemctl systeminfo timedatectl top tracepath tracepath6 traceroute traceroute6 ufw uname update-alternatives upower uptime vdir vmstat w wc who xrandr zipinfo) - jc_parsers=(--acpi --airport --airport-s --arp --asciitable --asciitable-m --blkid --cef --chage --cksum --crontab --crontab-u --csv --csv-s --date --df --dig --dir --dmidecode --dpkg-l --du --email-address --env --file --finger --free --fstab --git-log --git-log-s --gpg --group --gshadow --hash --hashsum --hciconfig --history --hosts --id --ifconfig --ini --iostat --iostat-s --ip-address --iptables --iso-datetime --iw-scan --jar-manifest --jobs --jwt --kv --last --ls --ls-s --lsblk --lsmod --lsof --lsusb --m3u --mdadm --mount --mpstat --mpstat-s --netstat --nmcli --ntpq --passwd --pidstat --pidstat-s --ping --ping-s --pip-list --pip-show --plist --postconf --ps --route --rpm-qi --rsync --rsync-s --sfdisk --shadow --ss --stat --stat-s --sysctl --syslog --syslog-s --syslog-bsd --syslog-bsd-s --systemctl --systemctl-lj --systemctl-ls --systemctl-luf --systeminfo --time --timedatectl --timestamp --top --top-s --tracepath --traceroute --ufw --ufw-appinfo --uname --update-alt-gs --update-alt-q --upower --uptime --url --vmstat --vmstat-s --w --wc --who --x509-cert --xml --xrandr --yaml --zipinfo) + jc_parsers=(--acpi --airport --airport-s --arp --asciitable --asciitable-m --blkid --cef --cef-s --chage --cksum --crontab --crontab-u --csv --csv-s --date --df --dig --dir --dmidecode --dpkg-l --du --email-address --env --file --finger --free --fstab --git-log --git-log-s --gpg --group --gshadow --hash --hashsum --hciconfig --history --hosts --id --ifconfig --ini --iostat --iostat-s --ip-address --iptables --iso-datetime --iw-scan --jar-manifest --jobs --jwt --kv --last --ls --ls-s --lsblk --lsmod --lsof --lsusb --m3u --mdadm --mount --mpstat --mpstat-s --netstat --nmcli --ntpq --passwd --pidstat --pidstat-s --ping --ping-s --pip-list --pip-show --plist --postconf --ps --route --rpm-qi --rsync --rsync-s --sfdisk --shadow --ss --stat --stat-s --sysctl --syslog --syslog-s --syslog-bsd --syslog-bsd-s --systemctl --systemctl-lj --systemctl-ls --systemctl-luf --systeminfo --time --timedatectl --timestamp --top --top-s --tracepath --traceroute --ufw --ufw-appinfo --uname --update-alt-gs --update-alt-q --upower --uptime --url --vmstat --vmstat-s --w --wc --who --x509-cert --xml --xrandr --yaml --zipinfo) jc_options=(--force-color -C --debug -d --monochrome -m --pretty -p --quiet -q --raw -r --time-out -t --unbuffer -u --yaml-out -y) jc_about_options=(--about -a) jc_about_mod_options=(--pretty -p --yaml-out -y --monochrome -m --force-color -C) diff --git a/completions/jc_zsh_completion.sh b/completions/jc_zsh_completion.sh index 206e5afc3..d89e4c7c9 100644 --- a/completions/jc_zsh_completion.sh +++ b/completions/jc_zsh_completion.sh @@ -95,7 +95,7 @@ _jc() { 'xrandr:run "xrandr" command with magic syntax.' 'zipinfo:run "zipinfo" command with magic syntax.' ) - jc_parsers=(--acpi --airport --airport-s --arp --asciitable --asciitable-m --blkid --cef --chage --cksum --crontab --crontab-u --csv --csv-s --date --df --dig --dir --dmidecode --dpkg-l --du --email-address --env --file --finger --free --fstab --git-log --git-log-s --gpg --group --gshadow --hash --hashsum --hciconfig --history --hosts --id --ifconfig --ini --iostat --iostat-s --ip-address --iptables --iso-datetime --iw-scan --jar-manifest --jobs --jwt --kv --last --ls --ls-s --lsblk --lsmod --lsof --lsusb --m3u --mdadm --mount --mpstat --mpstat-s --netstat --nmcli --ntpq --passwd --pidstat --pidstat-s --ping --ping-s --pip-list --pip-show --plist --postconf --ps --route --rpm-qi --rsync --rsync-s --sfdisk --shadow --ss --stat --stat-s --sysctl --syslog --syslog-s --syslog-bsd --syslog-bsd-s --systemctl --systemctl-lj --systemctl-ls --systemctl-luf --systeminfo --time --timedatectl --timestamp --top --top-s --tracepath --traceroute --ufw --ufw-appinfo --uname --update-alt-gs --update-alt-q --upower --uptime --url --vmstat --vmstat-s --w --wc --who --x509-cert --xml --xrandr --yaml --zipinfo) + jc_parsers=(--acpi --airport --airport-s --arp --asciitable --asciitable-m --blkid --cef --cef-s --chage --cksum --crontab --crontab-u --csv --csv-s --date --df --dig --dir --dmidecode --dpkg-l --du --email-address --env --file --finger --free --fstab --git-log --git-log-s --gpg --group --gshadow --hash --hashsum --hciconfig --history --hosts --id --ifconfig --ini --iostat --iostat-s --ip-address --iptables --iso-datetime --iw-scan --jar-manifest --jobs --jwt --kv --last --ls --ls-s --lsblk --lsmod --lsof --lsusb --m3u --mdadm --mount --mpstat --mpstat-s --netstat --nmcli --ntpq --passwd --pidstat --pidstat-s --ping --ping-s --pip-list --pip-show --plist --postconf --ps --route --rpm-qi --rsync --rsync-s --sfdisk --shadow --ss --stat --stat-s --sysctl --syslog --syslog-s --syslog-bsd --syslog-bsd-s --systemctl --systemctl-lj --systemctl-ls --systemctl-luf --systeminfo --time --timedatectl --timestamp --top --top-s --tracepath --traceroute --ufw --ufw-appinfo --uname --update-alt-gs --update-alt-q --upower --uptime --url --vmstat --vmstat-s --w --wc --who --x509-cert --xml --xrandr --yaml --zipinfo) jc_parsers_describe=( '--acpi:`acpi` command parser' '--airport:`airport -I` command parser' @@ -105,6 +105,7 @@ _jc() { '--asciitable-m:multi-line ASCII and Unicode table parser' '--blkid:`blkid` command parser' '--cef:CEF string parser' + '--cef-s:CEF string streaming parser' '--chage:`chage --list` command parser' '--cksum:`cksum` and `sum` command parser' '--crontab:`crontab` command and file parser' diff --git a/docs/parsers/cef.md b/docs/parsers/cef.md index 04e082ae0..c264cc544 100644 --- a/docs/parsers/cef.md +++ b/docs/parsers/cef.md @@ -69,7 +69,7 @@ See: https://www.microfocus.com/documentation/arcsight/arcsight-smartconnectors- Examples: - $ cef | jc --cef -p + $ cat cef.log | jc --cef -p [ { "deviceVendor": "Trend Micro", @@ -96,7 +96,7 @@ Examples: } ] - $ cef | jc --cef -p -r + $ cat cef.log | jc --cef -p -r [ { "deviceVendor": "Trend Micro", diff --git a/docs/parsers/cef_s.md b/docs/parsers/cef_s.md new file mode 100644 index 000000000..eb403ca68 --- /dev/null +++ b/docs/parsers/cef_s.md @@ -0,0 +1,124 @@ +[Home](https://kellyjonbrazil.github.io/jc/) + + +# jc.parsers.cef\_s + +jc - JSON Convert CEF string output streaming parser + +> This streaming parser outputs JSON Lines (cli) or returns an Iterable of +> Dictionaries (module) + +This parser conforms to the Microfocus Arcsight CEF specification. + +This parser will accept a single CEF string or multiple CEF string lines. +Any text before "CEF" will be ignored. Syslog and CEF escaped characters +(`\\`, `\\"`, `\\]`, `\\|`, `\\=`, `\\%`, `\\#`, `\\n`, and `\\r`) are +unescaped. + +Extended fields, as defined in the CEF specification, are relabeled +and the values are converted to their respective types. Extra naive and +UTC epoch timestamps are added where appropriate per the CEF specification. + +A warning message to `STDERR` will be printed if an unparsable line is found +unless `--quiet` or `quiet=True` is used. + +To preserve escaping and original keynames and to prevent type conversions +use the `--raw` CLI option or `raw=True` param in the `parse()` function. + +Usage (cli): + + $ echo 'CEF:0|Vendor|Product|3.2.0|1|SYSTEM|1|... | jc --cef-s + +Usage (module): + + import jc + + result = jc.parse('cef_s', cef_command_output.splitlines()) + for item in result: + # do something + +Schema: + +See: https://www.microfocus.com/documentation/arcsight/arcsight-smartconnectors-8.3/cef-implementation-standard/Content/CEF/Chapter%201%20What%20is%20CEF.htm + +> Note: Special characters in key names will be converted to underscores. + + { + "deviceVendor": string, + "deviceProduct": string, + "deviceVersion": string, + "deviceEventClassId": string, + "deviceEventClassIdNum": integer/null, + "name": string, + "agentSeverity": string/integer, + "agentSeverityString": string, + "agentSeverityNum": integer/null, + "CEFVersion": integer, + string/integer/float, # [0] + "_epoch": integer/null, # [1] + "_epoch_utc": integer/null, # [2] + string, + "unparsable": string # [3] + + # below object only exists if using -qq or ignore_exceptions=True + "_jc_meta": { + "success": boolean, # false if error parsing + "error": string, # exists if "success" is false + "line": string # exists if "success" is false + } + } + + [0] Will attempt to convert extended fields to the type specified in the + CEF specification. If conversion fails, then the field will remain + a string. + [1] Naive calculated epoch timestamp + [2] Timezone-aware calculated epoch timestamp. (UTC only) This value + will be null if a UTC timezone cannot be extracted from the original + timestamp string value. + [3] This field exists if the CEF line is not parsable. The value + is the original syslog line. + +Examples: + + $ cat cef.log | jc --cef-s + {"deviceVendor":"Fortinet","deviceProduct":"FortiDeceptor","deviceV...} + {"deviceVendor":"Trend Micro","deviceProduct":"Deep Security Agent"...} + ... + + $ cat cef.log | jc --cef-s -r + {"deviceVendor":"Fortinet","deviceProduct":"FortiDeceptor","deviceV...} + {"deviceVendor":"Trend Micro","deviceProduct":"Deep Security Agent"...} + ... + + + +### parse + +```python +@add_jc_meta +def parse(data: Iterable[str], + raw: bool = False, + quiet: bool = False, + ignore_exceptions: bool = False) -> Union[Iterable[Dict], tuple] +``` + +Main text parsing generator function. Returns an iterable object. + +Parameters: + + data: (iterable) line-based text data to parse + (e.g. sys.stdin or str.splitlines()) + + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + ignore_exceptions: (boolean) ignore parsing exceptions if True + + +Returns: + + Iterable of Dictionaries + +### Parser Information +Compatibility: linux, darwin, cygwin, win32, aix, freebsd + +Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/jc/lib.py b/jc/lib.py index 92026dbff..2675a43f9 100644 --- a/jc/lib.py +++ b/jc/lib.py @@ -17,6 +17,7 @@ 'asciitable-m', 'blkid', 'cef', + 'cef-s', 'chage', 'cksum', 'crontab', diff --git a/jc/parsers/cef.py b/jc/parsers/cef.py index 300bf0ace..44d4f1c5a 100644 --- a/jc/parsers/cef.py +++ b/jc/parsers/cef.py @@ -64,7 +64,7 @@ Examples: - $ cef | jc --cef -p + $ cat cef.log | jc --cef -p [ { "deviceVendor": "Trend Micro", @@ -91,7 +91,7 @@ } ] - $ cef | jc --cef -p -r + $ cat cef.log | jc --cef -p -r [ { "deviceVendor": "Trend Micro", diff --git a/jc/parsers/cef_s.py b/jc/parsers/cef_s.py new file mode 100644 index 000000000..88826689a --- /dev/null +++ b/jc/parsers/cef_s.py @@ -0,0 +1,320 @@ +"""jc - JSON Convert CEF string output streaming parser + +> This streaming parser outputs JSON Lines (cli) or returns an Iterable of +> Dictionaries (module) + +This parser conforms to the Microfocus Arcsight CEF specification. + +This parser will accept a single CEF string or multiple CEF string lines. +Any text before "CEF" will be ignored. Syslog and CEF escaped characters +(`\\`, `\\"`, `\\]`, `\\|`, `\\=`, `\\%`, `\\#`, `\\n`, and `\\r`) are +unescaped. + +Extended fields, as defined in the CEF specification, are relabeled +and the values are converted to their respective types. Extra naive and +UTC epoch timestamps are added where appropriate per the CEF specification. + +A warning message to `STDERR` will be printed if an unparsable line is found +unless `--quiet` or `quiet=True` is used. + +To preserve escaping and original keynames and to prevent type conversions +use the `--raw` CLI option or `raw=True` param in the `parse()` function. + +Usage (cli): + + $ echo 'CEF:0|Vendor|Product|3.2.0|1|SYSTEM|1|... | jc --cef-s + +Usage (module): + + import jc + + result = jc.parse('cef_s', cef_command_output.splitlines()) + for item in result: + # do something + +Schema: + +See: https://www.microfocus.com/documentation/arcsight/arcsight-smartconnectors-8.3/cef-implementation-standard/Content/CEF/Chapter%201%20What%20is%20CEF.htm + +> Note: Special characters in key names will be converted to underscores. + + { + "deviceVendor": string, + "deviceProduct": string, + "deviceVersion": string, + "deviceEventClassId": string, + "deviceEventClassIdNum": integer/null, + "name": string, + "agentSeverity": string/integer, + "agentSeverityString": string, + "agentSeverityNum": integer/null, + "CEFVersion": integer, + string/integer/float, # [0] + "_epoch": integer/null, # [1] + "_epoch_utc": integer/null, # [2] + string, + "unparsable": string # [3] + + # below object only exists if using -qq or ignore_exceptions=True + "_jc_meta": { + "success": boolean, # false if error parsing + "error": string, # exists if "success" is false + "line": string # exists if "success" is false + } + } + + [0] Will attempt to convert extended fields to the type specified in the + CEF specification. If conversion fails, then the field will remain + a string. + [1] Naive calculated epoch timestamp + [2] Timezone-aware calculated epoch timestamp. (UTC only) This value + will be null if a UTC timezone cannot be extracted from the original + timestamp string value. + [3] This field exists if the CEF line is not parsable. The value + is the original syslog line. + +Examples: + + $ cat cef.log | jc --cef-s + {"deviceVendor":"Fortinet","deviceProduct":"FortiDeceptor","deviceV...} + {"deviceVendor":"Trend Micro","deviceProduct":"Deep Security Agent"...} + ... + + $ cat cef.log | jc --cef-s -r + {"deviceVendor":"Fortinet","deviceProduct":"FortiDeceptor","deviceV...} + {"deviceVendor":"Trend Micro","deviceProduct":"Deep Security Agent"...} + ... +""" +from typing import Dict, Iterable, Union +import re +import jc.utils +from jc.streaming import ( + add_jc_meta, streaming_input_type_check, streaming_line_input_type_check, raise_or_yield +) +from jc.exceptions import ParseError +from jc.parsers.cef import _pycef_parse + + +class info(): + """Provides parser metadata (version, author, etc.)""" + version = '1.0' + description = 'CEF string streaming parser' + author = 'Kelly Brazil' + author_email = 'kellyjonbrazil@gmail.com' + compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd'] + streaming = True + + +__version__ = info.version + + +def _process(proc_data: Dict) -> Dict: + """ + Final processing to conform to the schema. + + Parameters: + + proc_data: (Dictionary) raw structured data to process + + Returns: + + Dictionary. Structured data to conform to the schema. + """ + # fix escape chars specified in syslog RFC 5424 and CEF spec + # https://www.rfc-editor.org/rfc/rfc5424.html#section-6 + # https://www.microfocus.com/documentation/arcsight/arcsight-smartconnectors-8.3/cef-implementation-standard/Content/CEF/Chapter%201%20What%20is%20CEF.htm?tocpath=_____2#_Toc494359738 + escape_map = { + r'\\': '\\', + r'\"': '"', + r'\]': ']', + r'\|': '|', + r'\=': '=', + r'\%': '%', + r'\#': '#', + r'\n': '\n', + r'\r': '\r' + } + + int_list = {'CEFVersion'} + + severity_map = { + None: 'Unknown', + 0: 'Low', + 1: 'Low', + 2: 'Low', + 3: 'Low', + 4: 'Medium', + 5: 'Medium', + 6: 'Medium', + 7: 'High', + 8: 'High', + 9: 'Very-High', + 10: 'Very-High' + } + + severity_set = {'unknown', 'low', 'medium', 'high', 'very-high'} + + # set defined types for extended fields + # see https://www.microfocus.com/documentation/arcsight/arcsight-smartconnectors-8.3/cef-implementation-standard/#CEF/Chapter%202%20ArcSight%20Extension.htm + extended_ints = { + 'spid', 'customerKey', 'deviceTranslatedZoneKey', 'oldFileSize', + 'destinationTranslatedPort', 'cn3', 'sourceTranslatedPort', 'in', 'fsize', 'slat', + 'dpid', 'cnt', 'agentZoneKey', 'out', 'type', 'eventId', 'dlong', 'cn2', + 'deviceDirection', 'spt', 'agentTranslatedZoneKey', 'sTranslatedZoneKey', 'cn1', + 'slong', 'dZoneKey', 'deviceZoneKey', 'dvcpid', 'dpt', 'dTranslatedZoneKey', 'dlat', + 'sZoneKey' + } + + extended_floats = { + 'cfp1', 'cfp2', 'cfp3', 'cfp4' + } + + extended_dt = { + 'deviceCustomDate1', 'deviceCustomDate2', 'end', 'fileCreateTime', + 'fileModificationTime', 'flexDate1', 'oldFileCreateTime', 'oldFileModificationTime', + 'rt', 'start', 'art' + } + + for key, value in proc_data.copy().items(): + if key in extended_ints: + try: + proc_data[key] = int(value) + except Exception: + pass + + if key in extended_floats: + try: + proc_data[key] = float(value) + except Exception: + pass + + if key in extended_dt: + if re.match(r'\d{10,13}', proc_data[key]): + proc_data[key + '_epoch'] = int(proc_data[key][:10]) + proc_data[key + '_epoch_utc'] = None + else: + formats = (1400, 1410, 1420, 1430) + dt = jc.utils.timestamp(proc_data[key], formats) + proc_data[key + '_epoch'] = dt.naive + proc_data[key + '_epoch_utc'] = dt.utc + + # Process custom field labels (adapted from pycef library) + cleanup_list = [] + custom_fields = list(proc_data.keys()) + for key in custom_fields: + if key.endswith('Label'): + customlabel = key[:-5] + for customfield in custom_fields: + new_name = proc_data[key] + # check for normal custom fields + if customfield == customlabel: + proc_data[new_name] = proc_data[customfield] + cleanup_list.append(customfield) + cleanup_list.append(key) + + # check for datetime objects + if customfield == customlabel + '_epoch': + proc_data[new_name + '_epoch'] = proc_data[customfield] + cleanup_list.append(customfield) + + if customfield == customlabel + '_epoch_utc': + proc_data[new_name + '_epoch_utc'] = proc_data[customfield] + cleanup_list.append(customfield) + + # cleanup extra custom fields + for key in cleanup_list: + del proc_data[key] + + # more normalization + for key, value in proc_data.copy().items(): + if isinstance(proc_data[key], str): + # remove any spaces around values + proc_data[key] = value.strip() + + # fixup escaped characters + for esc, esc_sub in escape_map.items(): + proc_data[key] = proc_data[key].replace(esc, esc_sub) + + # normalize keynames + new_key = key.strip() + new_key = re.sub(r'[^a-zA-Z0-9]', '_', new_key) + new_key = new_key.strip('_') + proc_data[new_key] = proc_data.pop(key) + + # integer conversions + if key in int_list: + proc_data[key] = jc.utils.convert_to_int(proc_data[key]) + + # set agentSeverityString and agentSeverityNum: + if 'agentSeverity' in proc_data: + if proc_data['agentSeverity'].lower() in severity_set: + proc_data['agentSeverityString'] = proc_data['agentSeverity'] + proc_data['agentSeverityNum'] = None + else: + try: + proc_data['agentSeverityString'] = severity_map[int(proc_data['agentSeverity'])] + proc_data['agentSeverityNum'] = int(proc_data['agentSeverity']) + except Exception: + pass + + # set deviceEventClassIdNum: + if 'deviceEventClassId' in proc_data: + proc_data['deviceEventClassIdNum'] = jc.utils.convert_to_int(proc_data['deviceEventClassId']) + + return proc_data + + +@add_jc_meta +def parse( + data: Iterable[str], + raw: bool = False, + quiet: bool = False, + ignore_exceptions: bool = False +) -> Union[Iterable[Dict], tuple]: + """ + Main text parsing generator function. Returns an iterable object. + + Parameters: + + data: (iterable) line-based text data to parse + (e.g. sys.stdin or str.splitlines()) + + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + ignore_exceptions: (boolean) ignore parsing exceptions if True + + + Returns: + + Iterable of Dictionaries + """ + jc.utils.compatibility(__name__, info.compatible, quiet) + streaming_input_type_check(data) + + for line in data: + try: + streaming_line_input_type_check(line) + output_line: Dict = {} + + #skip blank lines + if not line.strip(): + continue + + try: + output_line = _pycef_parse(line) + + except Exception: + output_line = { + 'unparsable': line.rstrip() + } + + if not quiet: + jc.utils.warning_message( + [f'Unparsable line found: {line.rstrip()}'] + ) + + if output_line: + yield output_line if raw else _process(output_line) + + except Exception as e: + yield raise_or_yield(ignore_exceptions, e, line) diff --git a/man/jc.1 b/man/jc.1 index f101dc077..002d11a08 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -57,6 +57,11 @@ multi-line ASCII and Unicode table parser \fB--cef\fP CEF string parser +.TP +.B +\fB--cef-s\fP +CEF string streaming parser + .TP .B \fB--chage\fP From f7c6a82e7396f0cd9281a81d89fa9971d59c22a4 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sat, 20 Aug 2022 13:38:20 -0700 Subject: [PATCH 111/116] add cef tests --- tests/fixtures/generic/cef-streaming.json | 1 + tests/fixtures/generic/cef.json | 1 + tests/fixtures/generic/cef.out | 23 ++++++++++++++ tests/test_cef.py | 35 +++++++++++++++++++++ tests/test_cef_s.py | 37 +++++++++++++++++++++++ 5 files changed, 97 insertions(+) create mode 100644 tests/fixtures/generic/cef-streaming.json create mode 100644 tests/fixtures/generic/cef.json create mode 100644 tests/fixtures/generic/cef.out create mode 100644 tests/test_cef.py create mode 100644 tests/test_cef_s.py diff --git a/tests/fixtures/generic/cef-streaming.json b/tests/fixtures/generic/cef-streaming.json new file mode 100644 index 000000000..3343b452c --- /dev/null +++ b/tests/fixtures/generic/cef-streaming.json @@ -0,0 +1 @@ +[{"deviceVendor":"Fortinet","deviceProduct":"FortiDeceptor","deviceVersion":"3.2.0","deviceEventClassId":"1","name":"SYSTEM","agentSeverity":"1","CEFVersion":0,"date":"2020-12-08","time":"16:59:33","logid":"0136000001","type":"event","subtype":"attack","level":"alert","user":"system","ui":"GUI","action":"Incident_Detection","status":"success","reason":"none","EventID":"1845921387423247329","IncidentID":"1845921507147395878","Tagkey":"192.168.100.1:59840:192.168.100.21:1836840592250413230","AttackerIP":"192.168.100.1","AttackerPort":"59840","VictimIP":"192.168.100.21","VictimPort":"445","Operation":"Logon_via_net_share","Service":"SAMBA","Username":"glen","Password":"lovely","Description":"\"SAMBA Login with password: lovely\"\"","agentSeverityString":"Low","agentSeverityNum":1,"deviceEventClassIdNum":1},{"deviceVendor":"Fortinet","deviceProduct":"FortiDeceptor","deviceVersion":"3.2.0","deviceEventClassId":"1","name":"SYSTEM","agentSeverity":"1","CEFVersion":0,"date":"2020-12-08","time":"16:59:33","logid":"0136000001","type":"event","subtype":"attack","level":"alert","user":"system","ui":"GUI","action":"Incident_Detection","status":"success","reason":"none","EventID":"1845921387423247329","IncidentID":"1845921507147395878","Tagkey":"192.168.100.1:59840:192.168.100.21:1836840592250413230","AttackerIP":"192.168.100.1","AttackerPort":"59840","VictimIP":"192.168.100.21","VictimPort":"445","Operation":"Logon_via_net_share","Service":"SAMBA","Username":"glen","Password":"lovely","Description":"\"this is a description\"\"","agentSeverityString":"Low","agentSeverityNum":1,"deviceEventClassIdNum":1},{"deviceVendor":"Trend Micro","deviceProduct":"Deep Security Agent","deviceVersion":"","deviceEventClassId":"4000000","name":"Eicar_test_file","agentSeverity":"6","CEFVersion":0,"dvchost":"hostname","string":"hello \"world\" this is a backslash: \\ and this is a bracket ]!","another":"field","Host_ID":1,"Quarantine":205,"agentSeverityString":"Medium","agentSeverityNum":6,"deviceEventClassIdNum":4000000},{"deviceVendor":"Trend Micro","deviceProduct":"Deep Security Agent","deviceVersion":"","deviceEventClassId":"4000000","name":"Eicar_test_file","agentSeverity":"Medium","CEFVersion":0,"dvchost":"hostname","filePath":"C:\\Users\\trend\\Desktop\\eicar.exe","act":"Delete","result":"Delete","msg":"Realtime","TrendMicroDsMalwareTarget":"N/A","N_TrendMicroDsFileMD5":"44D88612FEA8A8F36DE82E1278ABB02F","TrendMicroDsFileSHA1":"3395856CE81F2B7382DEE72602F798B642F14140","TrendMicroDsFileSHA256":"275A021BBFB6489E54D471899F7DB9D1663FC695EC2FE2A2C4538AABF651FD0F","TrendMicroDsDetectionConfidence":"95","TrendMicroDsRelevantDetectionNames":"Ransom_CERBER.BZC;Ransom_CERBER.C;Ransom_CRYPNISCA.SM","Host_ID":1,"Quarantine_File_Size":205,"Container":"ContainerImageName | ContainerName | ContainerID","agentSeverityString":"Medium","agentSeverityNum":null,"deviceEventClassIdNum":4000000},{"deviceVendor":"Trend Micro","deviceProduct":"Deep Security Agent","deviceVersion":"","deviceEventClassId":"4000000","name":"Eicar_test_file","agentSeverity":"6","CEFVersion":0,"dvchost":"hostname","string":"hello \"world\" this is a backslash: \\ and this is a bracket ] this is equal =, this is pipe |, this is newline \n and another newline \n the end!","another":"field","Host_ID":1,"Quarantine":205,"agentSeverityString":"Medium","agentSeverityNum":6,"deviceEventClassIdNum":4000000},{"deviceVendor":"Trend Micro","deviceProduct":"Deep Security Agent","deviceVersion":"","deviceEventClassId":"4000000","name":"Eicar_test_file","agentSeverity":"6","CEFVersion":0,"dvchost":"hostname","string":"hello \"world\" this is a backslash: \\ and this is a bracket ]!","another":"field","start":"Nov 08 2020 12:30:00.111 UTC","start_epoch":1604867400,"start_epoch_utc":1604838600,"Host_ID":1,"Quarantine":205,"myDate":"Nov 08 2022 12:30:00.111","myDate_epoch":1667939400,"myDate_epoch_utc":null,"myFloat":3.14,"myTimestampDate":"1660966164045","myTimestampDate_epoch":1660966164,"myTimestampDate_epoch_utc":null,"agentSeverityString":"Medium","agentSeverityNum":6,"deviceEventClassIdNum":4000000},{"deviceVendor":"Incapsula","deviceProduct":"SIEMintegration","deviceVersion":"1","deviceEventClassId":"1","name":"Illegal Resource Access","agentSeverity":"3","CEFVersion":0,"fileid":"3412341160002518171","sourceServiceName":"site123.abcd.info","siteid":"1509732","suid":"50005477","requestClientApplication":"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.0","deviceFacility":"mia","ccode":"IL","tag":"www.elvis.com","cn1":200,"in":54,"xff":"44.44.44.44","dproc":"Browser","cicode":"Rehovot","Customer":"CEFcustomer123","siteTag":"my-site-tag","start":"1453290121336","request":"site123.abcd.info/","requestmethod":"GET","qstr":"p=%2fetc%2fpasswd","app":"HTTP","act":"REQ_CHALLENGE_CAPTCHA","deviceExternalID":"33411452762204224","cpt":"443","src":"12.12.12.12","ver":"TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256","end":"1566300670892","additionalReqHeaders":"[{\"Accept\":\"*/*\"},{\"x-v\":\"1\"},{\"x-fapi-interaction-id\":\"10.10.10.10\"}]","additionalResHeaders":"[{\"Content-Type\":\"text/html; charset=UTF-8\"}]","filetype":"30037,1001,","filepermission":"2,1,","start_epoch":1453290121,"start_epoch_utc":null,"end_epoch":1566300670,"end_epoch_utc":null,"Javascript_Support":"true","CO_Support":"true","Cap_Support":"NOT_SUPPORTED","VID":"c2e72124-0e8a-4dd8-b13b-3da246af3ab2","clappsig":"de3c633ac428e0678f3aac20cf7f239431e54cbb8a17e8302f53653923305e1835a9cd871db32aa4fc7b8a9463366cc4","clapp":"Firefox","latitude":"31.8969","longitude":"34.8186","Rule_name":"Block Malicious User,High Risk Resources,","Rule_Additional_Info":",,[{\"api_specification_violation_type\":\"INVALID_PARAM_NAME\",\"parameter_name\":\"somename\"}]","agentSeverityString":"Low","agentSeverityNum":3,"deviceEventClassIdNum":1},{"deviceVendor":"Incapsula","deviceProduct":"SIEMintegration","deviceVersion":"1","deviceEventClassId":"1","name":"Normal","agentSeverity":"0","CEFVersion":0,"sourceServiceName":"site123.abcd.info","siteid":"1509732","suid":"50005477","requestClientApplication":"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.0","deviceFacility":"mia","ccode":"IL","tag":"www.elvis.com","cicode":"Rehovot","Customer":"CEFcustomer123","siteTag":"my-site-tag","start":"1453290121336","request":"site123.abcd.info/main.css","ref":"www.incapsula.com/lama","requestmethod":"GET","cn1":200,"app":"HTTP","deviceExternalID":"33411452762204224","in":54,"xff":"44.44.44.44","cpt":"443","src":"12.12.12.12","ver":"TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256","end":"1566300670892","additionalReqHeaders":"[{\"Accept\":\"*/*\"},{\"x-v\":\"1\"},{\"x-fapi-interaction-id\":\"10.10.10.10\"}]","additionalResHeaders":"[{\"Content-Type\":\"text/html; charset=UTF-8\"}]","start_epoch":1453290121,"start_epoch_utc":null,"end_epoch":1566300670,"end_epoch_utc":null,"latitude":"31.8969","longitude":"34.8186","agentSeverityString":"Low","agentSeverityNum":0,"deviceEventClassIdNum":1},{"deviceVendor":"Incapsula","deviceProduct":"SIEMintegration","deviceVersion":"1","deviceEventClassId":"my device id","name":"Normal","agentSeverity":"0","CEFVersion":0,"sourceServiceName":"site123.abcd.info","siteid":"1509732","suid":"50005477","requestClientApplication":"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.0","deviceFacility":"mia","ccode":"IL","tag":"www.elvis.com","cicode":"Rehovot","Customer":"CEFcustomer123","siteTag":"my-site-tag","start":"1453290121336","request":"site123.abcd.info/main.css","ref":"www.incapsula.com/lama","requestmethod":"GET","cn1":200,"app":"HTTP","deviceExternalID":"33411452762204224","in":54,"xff":"44.44.44.44","cpt":"443","src":"12.12.12.12","ver":"TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256","end":"1566300670892","additionalReqHeaders":"[{\"Accept\":\"*/*\"},{\"x-v\":\"1\"},{\"x-fapi-interaction-id\":\"10.10.10.10\"}]","additionalResHeaders":"[{\"Content-Type\":\"text/html; charset=UTF-8\"}]","start_epoch":1453290121,"start_epoch_utc":null,"end_epoch":1566300670,"end_epoch_utc":null,"latitude":"31.8969","longitude":"34.8186","agentSeverityString":"Low","agentSeverityNum":0,"deviceEventClassIdNum":null},{"deviceVendor":"Kaspersky Lab","deviceProduct":"Kaspersky ICAP Server","deviceVersion":"%VERSION%","deviceEventClassId":"%EVENT_CLASS_ID%","name":"%EVENT_NAME%","agentSeverity":"%SEVERITY%","CEFVersion":0,"msg":"%EVENT_MSG%","src":"%CLIENT_IP%","dvcpid":"%ICAP_SERVER_PID%","start":"%EVENT_TIME%","fileHash":"%SCANNED_FILE_HASH%","request":"%SCANNED_URL%","start_epoch":null,"start_epoch_utc":null,"X_Client_Username":"%HTTP_USER_NAME%","X_Client_IP":"%HTTP_USER_IP%","Scan_result":"%SCAN_RESULT%","Virus_name":"%VIRUS_NAME%","SHA256":"%SCANNED_FILE_SHA256_HASH%","deviceEventClassIdNum":null},{"deviceVendor":"Elastic","deviceProduct":"Vaporware","deviceVersion":"1.0.0-alpha","deviceEventClassId":"18","name":"Web request","agentSeverity":"low","CEFVersion":0,"eventId":3457,"msg":"hello","agentSeverityString":"low","agentSeverityNum":null,"deviceEventClassIdNum":18},{"deviceVendor":"Aruba Networks","deviceProduct":"ClearPass","deviceVersion":"6.5.0.69058","deviceEventClassId":"0-1-0","name":"Insight Logs","agentSeverity":"0","CEFVersion":0,"Auth_Username":"host/Asif-Test-PC2","Auth_Authorization_Sources":"null","Auth_Login_Status":"216","Auth_Request_Timestamp":"2017-12-03 16:28:20+05:30","Auth_Protocol":"RADIUS","Auth_Source":"null","Auth_Enforcement_Profiles":"[Allow Access Profile]","Auth_NAS_Port":"null","Auth_SSID":"cppm-dot1x-test","TimestampFormat":"MMM dd yyyy HH:mm:ss.SSS zzz","Auth_NAS_Port_Type":"19","Auth_Error_Code":"216","Auth_Roles":"null","Auth_Service":"Test Wireless","Auth_Host_MAC_Address":"6817294b0636","Auth_Unhealthy":"null","Auth_NAS_IP_Address":"10.17.4.7","src":"10.17.4.208","Auth_CalledStationId":"000B8661CD70","Auth_NAS_Identifier":"ClearPassLab3600","agentSeverityString":"Low","agentSeverityNum":0,"deviceEventClassIdNum":null},{"unparsable":"unparsable line"},{"deviceVendor":"Aruba Networks","deviceProduct":"ClearPass","deviceVersion":"6.5.0.68754","deviceEventClassId":"13-1-0","name":"Audit Records","agentSeverity":"5","CEFVersion":0,"cat":"Role","timeFormat":"MMM dd yyyy HH:mm:ss.SSS zzz","rt":"Nov 19, 2014 18:21:13 IST","src":"Test Role 10","act":"ADD","usrName":"admin","rt_epoch":null,"rt_epoch_utc":null,"agentSeverityString":"Medium","agentSeverityNum":5,"deviceEventClassIdNum":null}] diff --git a/tests/fixtures/generic/cef.json b/tests/fixtures/generic/cef.json new file mode 100644 index 000000000..3343b452c --- /dev/null +++ b/tests/fixtures/generic/cef.json @@ -0,0 +1 @@ +[{"deviceVendor":"Fortinet","deviceProduct":"FortiDeceptor","deviceVersion":"3.2.0","deviceEventClassId":"1","name":"SYSTEM","agentSeverity":"1","CEFVersion":0,"date":"2020-12-08","time":"16:59:33","logid":"0136000001","type":"event","subtype":"attack","level":"alert","user":"system","ui":"GUI","action":"Incident_Detection","status":"success","reason":"none","EventID":"1845921387423247329","IncidentID":"1845921507147395878","Tagkey":"192.168.100.1:59840:192.168.100.21:1836840592250413230","AttackerIP":"192.168.100.1","AttackerPort":"59840","VictimIP":"192.168.100.21","VictimPort":"445","Operation":"Logon_via_net_share","Service":"SAMBA","Username":"glen","Password":"lovely","Description":"\"SAMBA Login with password: lovely\"\"","agentSeverityString":"Low","agentSeverityNum":1,"deviceEventClassIdNum":1},{"deviceVendor":"Fortinet","deviceProduct":"FortiDeceptor","deviceVersion":"3.2.0","deviceEventClassId":"1","name":"SYSTEM","agentSeverity":"1","CEFVersion":0,"date":"2020-12-08","time":"16:59:33","logid":"0136000001","type":"event","subtype":"attack","level":"alert","user":"system","ui":"GUI","action":"Incident_Detection","status":"success","reason":"none","EventID":"1845921387423247329","IncidentID":"1845921507147395878","Tagkey":"192.168.100.1:59840:192.168.100.21:1836840592250413230","AttackerIP":"192.168.100.1","AttackerPort":"59840","VictimIP":"192.168.100.21","VictimPort":"445","Operation":"Logon_via_net_share","Service":"SAMBA","Username":"glen","Password":"lovely","Description":"\"this is a description\"\"","agentSeverityString":"Low","agentSeverityNum":1,"deviceEventClassIdNum":1},{"deviceVendor":"Trend Micro","deviceProduct":"Deep Security Agent","deviceVersion":"","deviceEventClassId":"4000000","name":"Eicar_test_file","agentSeverity":"6","CEFVersion":0,"dvchost":"hostname","string":"hello \"world\" this is a backslash: \\ and this is a bracket ]!","another":"field","Host_ID":1,"Quarantine":205,"agentSeverityString":"Medium","agentSeverityNum":6,"deviceEventClassIdNum":4000000},{"deviceVendor":"Trend Micro","deviceProduct":"Deep Security Agent","deviceVersion":"","deviceEventClassId":"4000000","name":"Eicar_test_file","agentSeverity":"Medium","CEFVersion":0,"dvchost":"hostname","filePath":"C:\\Users\\trend\\Desktop\\eicar.exe","act":"Delete","result":"Delete","msg":"Realtime","TrendMicroDsMalwareTarget":"N/A","N_TrendMicroDsFileMD5":"44D88612FEA8A8F36DE82E1278ABB02F","TrendMicroDsFileSHA1":"3395856CE81F2B7382DEE72602F798B642F14140","TrendMicroDsFileSHA256":"275A021BBFB6489E54D471899F7DB9D1663FC695EC2FE2A2C4538AABF651FD0F","TrendMicroDsDetectionConfidence":"95","TrendMicroDsRelevantDetectionNames":"Ransom_CERBER.BZC;Ransom_CERBER.C;Ransom_CRYPNISCA.SM","Host_ID":1,"Quarantine_File_Size":205,"Container":"ContainerImageName | ContainerName | ContainerID","agentSeverityString":"Medium","agentSeverityNum":null,"deviceEventClassIdNum":4000000},{"deviceVendor":"Trend Micro","deviceProduct":"Deep Security Agent","deviceVersion":"","deviceEventClassId":"4000000","name":"Eicar_test_file","agentSeverity":"6","CEFVersion":0,"dvchost":"hostname","string":"hello \"world\" this is a backslash: \\ and this is a bracket ] this is equal =, this is pipe |, this is newline \n and another newline \n the end!","another":"field","Host_ID":1,"Quarantine":205,"agentSeverityString":"Medium","agentSeverityNum":6,"deviceEventClassIdNum":4000000},{"deviceVendor":"Trend Micro","deviceProduct":"Deep Security Agent","deviceVersion":"","deviceEventClassId":"4000000","name":"Eicar_test_file","agentSeverity":"6","CEFVersion":0,"dvchost":"hostname","string":"hello \"world\" this is a backslash: \\ and this is a bracket ]!","another":"field","start":"Nov 08 2020 12:30:00.111 UTC","start_epoch":1604867400,"start_epoch_utc":1604838600,"Host_ID":1,"Quarantine":205,"myDate":"Nov 08 2022 12:30:00.111","myDate_epoch":1667939400,"myDate_epoch_utc":null,"myFloat":3.14,"myTimestampDate":"1660966164045","myTimestampDate_epoch":1660966164,"myTimestampDate_epoch_utc":null,"agentSeverityString":"Medium","agentSeverityNum":6,"deviceEventClassIdNum":4000000},{"deviceVendor":"Incapsula","deviceProduct":"SIEMintegration","deviceVersion":"1","deviceEventClassId":"1","name":"Illegal Resource Access","agentSeverity":"3","CEFVersion":0,"fileid":"3412341160002518171","sourceServiceName":"site123.abcd.info","siteid":"1509732","suid":"50005477","requestClientApplication":"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.0","deviceFacility":"mia","ccode":"IL","tag":"www.elvis.com","cn1":200,"in":54,"xff":"44.44.44.44","dproc":"Browser","cicode":"Rehovot","Customer":"CEFcustomer123","siteTag":"my-site-tag","start":"1453290121336","request":"site123.abcd.info/","requestmethod":"GET","qstr":"p=%2fetc%2fpasswd","app":"HTTP","act":"REQ_CHALLENGE_CAPTCHA","deviceExternalID":"33411452762204224","cpt":"443","src":"12.12.12.12","ver":"TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256","end":"1566300670892","additionalReqHeaders":"[{\"Accept\":\"*/*\"},{\"x-v\":\"1\"},{\"x-fapi-interaction-id\":\"10.10.10.10\"}]","additionalResHeaders":"[{\"Content-Type\":\"text/html; charset=UTF-8\"}]","filetype":"30037,1001,","filepermission":"2,1,","start_epoch":1453290121,"start_epoch_utc":null,"end_epoch":1566300670,"end_epoch_utc":null,"Javascript_Support":"true","CO_Support":"true","Cap_Support":"NOT_SUPPORTED","VID":"c2e72124-0e8a-4dd8-b13b-3da246af3ab2","clappsig":"de3c633ac428e0678f3aac20cf7f239431e54cbb8a17e8302f53653923305e1835a9cd871db32aa4fc7b8a9463366cc4","clapp":"Firefox","latitude":"31.8969","longitude":"34.8186","Rule_name":"Block Malicious User,High Risk Resources,","Rule_Additional_Info":",,[{\"api_specification_violation_type\":\"INVALID_PARAM_NAME\",\"parameter_name\":\"somename\"}]","agentSeverityString":"Low","agentSeverityNum":3,"deviceEventClassIdNum":1},{"deviceVendor":"Incapsula","deviceProduct":"SIEMintegration","deviceVersion":"1","deviceEventClassId":"1","name":"Normal","agentSeverity":"0","CEFVersion":0,"sourceServiceName":"site123.abcd.info","siteid":"1509732","suid":"50005477","requestClientApplication":"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.0","deviceFacility":"mia","ccode":"IL","tag":"www.elvis.com","cicode":"Rehovot","Customer":"CEFcustomer123","siteTag":"my-site-tag","start":"1453290121336","request":"site123.abcd.info/main.css","ref":"www.incapsula.com/lama","requestmethod":"GET","cn1":200,"app":"HTTP","deviceExternalID":"33411452762204224","in":54,"xff":"44.44.44.44","cpt":"443","src":"12.12.12.12","ver":"TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256","end":"1566300670892","additionalReqHeaders":"[{\"Accept\":\"*/*\"},{\"x-v\":\"1\"},{\"x-fapi-interaction-id\":\"10.10.10.10\"}]","additionalResHeaders":"[{\"Content-Type\":\"text/html; charset=UTF-8\"}]","start_epoch":1453290121,"start_epoch_utc":null,"end_epoch":1566300670,"end_epoch_utc":null,"latitude":"31.8969","longitude":"34.8186","agentSeverityString":"Low","agentSeverityNum":0,"deviceEventClassIdNum":1},{"deviceVendor":"Incapsula","deviceProduct":"SIEMintegration","deviceVersion":"1","deviceEventClassId":"my device id","name":"Normal","agentSeverity":"0","CEFVersion":0,"sourceServiceName":"site123.abcd.info","siteid":"1509732","suid":"50005477","requestClientApplication":"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.0","deviceFacility":"mia","ccode":"IL","tag":"www.elvis.com","cicode":"Rehovot","Customer":"CEFcustomer123","siteTag":"my-site-tag","start":"1453290121336","request":"site123.abcd.info/main.css","ref":"www.incapsula.com/lama","requestmethod":"GET","cn1":200,"app":"HTTP","deviceExternalID":"33411452762204224","in":54,"xff":"44.44.44.44","cpt":"443","src":"12.12.12.12","ver":"TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256","end":"1566300670892","additionalReqHeaders":"[{\"Accept\":\"*/*\"},{\"x-v\":\"1\"},{\"x-fapi-interaction-id\":\"10.10.10.10\"}]","additionalResHeaders":"[{\"Content-Type\":\"text/html; charset=UTF-8\"}]","start_epoch":1453290121,"start_epoch_utc":null,"end_epoch":1566300670,"end_epoch_utc":null,"latitude":"31.8969","longitude":"34.8186","agentSeverityString":"Low","agentSeverityNum":0,"deviceEventClassIdNum":null},{"deviceVendor":"Kaspersky Lab","deviceProduct":"Kaspersky ICAP Server","deviceVersion":"%VERSION%","deviceEventClassId":"%EVENT_CLASS_ID%","name":"%EVENT_NAME%","agentSeverity":"%SEVERITY%","CEFVersion":0,"msg":"%EVENT_MSG%","src":"%CLIENT_IP%","dvcpid":"%ICAP_SERVER_PID%","start":"%EVENT_TIME%","fileHash":"%SCANNED_FILE_HASH%","request":"%SCANNED_URL%","start_epoch":null,"start_epoch_utc":null,"X_Client_Username":"%HTTP_USER_NAME%","X_Client_IP":"%HTTP_USER_IP%","Scan_result":"%SCAN_RESULT%","Virus_name":"%VIRUS_NAME%","SHA256":"%SCANNED_FILE_SHA256_HASH%","deviceEventClassIdNum":null},{"deviceVendor":"Elastic","deviceProduct":"Vaporware","deviceVersion":"1.0.0-alpha","deviceEventClassId":"18","name":"Web request","agentSeverity":"low","CEFVersion":0,"eventId":3457,"msg":"hello","agentSeverityString":"low","agentSeverityNum":null,"deviceEventClassIdNum":18},{"deviceVendor":"Aruba Networks","deviceProduct":"ClearPass","deviceVersion":"6.5.0.69058","deviceEventClassId":"0-1-0","name":"Insight Logs","agentSeverity":"0","CEFVersion":0,"Auth_Username":"host/Asif-Test-PC2","Auth_Authorization_Sources":"null","Auth_Login_Status":"216","Auth_Request_Timestamp":"2017-12-03 16:28:20+05:30","Auth_Protocol":"RADIUS","Auth_Source":"null","Auth_Enforcement_Profiles":"[Allow Access Profile]","Auth_NAS_Port":"null","Auth_SSID":"cppm-dot1x-test","TimestampFormat":"MMM dd yyyy HH:mm:ss.SSS zzz","Auth_NAS_Port_Type":"19","Auth_Error_Code":"216","Auth_Roles":"null","Auth_Service":"Test Wireless","Auth_Host_MAC_Address":"6817294b0636","Auth_Unhealthy":"null","Auth_NAS_IP_Address":"10.17.4.7","src":"10.17.4.208","Auth_CalledStationId":"000B8661CD70","Auth_NAS_Identifier":"ClearPassLab3600","agentSeverityString":"Low","agentSeverityNum":0,"deviceEventClassIdNum":null},{"unparsable":"unparsable line"},{"deviceVendor":"Aruba Networks","deviceProduct":"ClearPass","deviceVersion":"6.5.0.68754","deviceEventClassId":"13-1-0","name":"Audit Records","agentSeverity":"5","CEFVersion":0,"cat":"Role","timeFormat":"MMM dd yyyy HH:mm:ss.SSS zzz","rt":"Nov 19, 2014 18:21:13 IST","src":"Test Role 10","act":"ADD","usrName":"admin","rt_epoch":null,"rt_epoch_utc":null,"agentSeverityString":"Medium","agentSeverityNum":5,"deviceEventClassIdNum":null}] diff --git a/tests/fixtures/generic/cef.out b/tests/fixtures/generic/cef.out new file mode 100644 index 000000000..4e986b0c1 --- /dev/null +++ b/tests/fixtures/generic/cef.out @@ -0,0 +1,23 @@ +CEF:0|Fortinet|FortiDeceptor|3.2.0|1|SYSTEM|1|date=2020-12-08 time=16:59:33 logid=0136000001 type=event subtype=attack level=alert user=system ui=GUI action=Incident_Detection status=success reason=none msg="EventID=1845921387423247329 IncidentID=1845921507147395878 Tagkey=192.168.100.1:59840:192.168.100.21:1836840592250413230 AttackerIP=192.168.100.1 AttackerPort=59840 VictimIP=192.168.100.21 VictimPort=445 Operation=Logon_via_net_share Service=SAMBA Username=glen Password=lovely Description=\"SAMBA Login with password: lovely\"" +CEF:0|Fortinet|FortiDeceptor|3.2.0|1|SYSTEM|1|date=2020-12-08 time=16:59:33 logid=0136000001 type=event subtype=attack level=alert user=system ui=GUI action=Incident_Detection status=success reason=none msg="EventID=1845921387423247329 IncidentID=1845921507147395878 Tagkey=192.168.100.1:59840:192.168.100.21:1836840592250413230 AttackerIP=192.168.100.1 AttackerPort=59840 VictimIP=192.168.100.21 VictimPort=445 Operation=Logon_via_net_share Service=SAMBA Username=glen Password=lovely Description=\"this is a description\"" +CEF:0|Trend Micro|Deep Security Agent||4000000|Eicar_test_file|6|cn1=1 cn1Label=Host ID dvchost=hostname cn2=205 cn2Label=Quarantine string=hello \"world\" this is a backslash: \\ and this is a bracket \]! another=field +CEF:0|Trend Micro|Deep Security Agent||4000000|Eicar_test_file|Medium|cn1=1 cn1Label=Host ID dvchost=hostname cn2=205 cn2Label=Quarantine File Size cs6=ContainerImageName | ContainerName | ContainerID cs6Label=Container filePath=C:\Users\trend\Desktop\eicar.exe act=Delete result=Delete msg=Realtime TrendMicroDsMalwareTarget=N/A TrendMicroDsMalwareTargetType=N/TrendMicroDsFileMD5=44D88612FEA8A8F36DE82E1278ABB02F TrendMicroDsFileSHA1=3395856CE81F2B7382DEE72602F798B642F14140 TrendMicroDsFileSHA256=275A021BBFB6489E54D471899F7DB9D1663FC695EC2FE2A2C4538AABF651FD0F TrendMicroDsDetectionConfidence=95 TrendMicroDsRelevantDetectionNames=Ransom_CERBER.BZC;Ransom_CERBER.C;Ransom_CRYPNISCA.SM +CEF:0|Trend Micro|Deep Security Agent||4000000|Eicar_test_file|6|cn1=1 cn1Label=Host ID dvchost=hostname cn2=205 cn2Label=Quarantine string=hello \"world\" this is a backslash: \\ and this is a bracket \] this is equal \=, this is pipe \|, this is newline \n and another newline \n the end! another=field + +CEF:0|Trend Micro|Deep Security Agent||4000000|Eicar_test_file|6|cn1=1 cn1Label=Host ID dvchost=hostname cn2=205 cn2Label=Quarantine string=hello \"world\" this is a backslash: \\ and this is a bracket \]! another=field start=Nov 08 2020 12:30:00.111 UTC deviceCustomDate1=Nov 08 2022 12:30:00.111 deviceCustomDate1Label=myDate cfp1=3.14 cfp1Label=myFloat deviceCustomDate2=1660966164045 deviceCustomDate2Label=myTimestampDate + +CEF:0|Incapsula|SIEMintegration|1|1|Illegal Resource Access|3| fileid=3412341160002518171 sourceServiceName=site123.abcd.info siteid=1509732 suid=50005477 requestClientApplication=Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.0 deviceFacility=mia cs2=true cs2Label=Javascript Support cs3=true cs3Label=CO Support ccode=IL tag=www.elvis.com cn1=200 in=54 xff=44.44.44.44 cs1=NOT_SUPPORTED cs1Label=Cap Support cs4=c2e72124-0e8a-4dd8-b13b-3da246af3ab2 cs4Label=VID cs5=de3c633ac428e0678f3aac20cf7f239431e54cbb8a17e8302f53653923305e1835a9cd871db32aa4fc7b8a9463366cc4 cs5Label=clappsig dproc=Browser cs6=Firefox cs6Label=clapp ccode=IL cicode=Rehovot cs7=31.8969 cs7Label=latitude cs8=34.8186 cs8Label=longitude Customer=CEFcustomer123 siteTag=my-site-tag start=1453290121336 request=site123.abcd.info/ requestmethod=GET qstr=p\=%2fetc%2fpasswd app=HTTP act=REQ_CHALLENGE_CAPTCHA deviceExternalID=33411452762204224 cpt=443 src=12.12.12.12 ver=TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 end=1566300670892 additionalReqHeaders=[{"Accept":"*/*"},{"x-v":"1"},{"x-fapi-interaction-id":"10.10.10.10"}] additionalResHeaders=[{"Content-Type":"text/html; charset\=UTF-8"}] filetype=30037,1001, filepermission=2,1, cs9=Block Malicious User,High Risk Resources, cs9Label=Rule name cs11=,,[{"api_specification_violation_type":"INVALID_PARAM_NAME","parameter_name":"somename"}] cs11Label=Rule Additional Info + +CEF:0|Incapsula|SIEMintegration|1|1|Normal|0| sourceServiceName=site123.abcd.info siteid=1509732 suid=50005477 requestClientApplication=Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.0 deviceFacility=mia ccode=IL tag=www.elvis.com cicode=Rehovot cs7=31.8969 cs7Label=latitude cs8=34.8186 cs8Label=longitude Customer=CEFcustomer123 siteTag=my-site-tag start=1453290121336 request=site123.abcd.info/main.css ref=www.incapsula.com/lama requestmethod=GET cn1=200 app=HTTP deviceExternalID=33411452762204224 in=54 xff=44.44.44.44 cpt=443 src=12.12.12.12 ver=TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 end=1566300670892 additionalReqHeaders=[{"Accept":"*/*"},{"x-v":"1"},{"x-fapi-interaction-id":"10.10.10.10"}] additionalResHeaders=[{"Content-Type":"text/html; charset\=UTF-8"}] + +CEF:0|Incapsula|SIEMintegration|1|my device id|Normal|0| sourceServiceName=site123.abcd.info siteid=1509732 suid=50005477 requestClientApplication=Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.0 deviceFacility=mia ccode=IL tag=www.elvis.com cicode=Rehovot cs7=31.8969 cs7Label=latitude cs8=34.8186 cs8Label=longitude Customer=CEFcustomer123 siteTag=my-site-tag start=1453290121336 request=site123.abcd.info/main.css ref=www.incapsula.com/lama requestmethod=GET cn1=200 app=HTTP deviceExternalID=33411452762204224 in=54 xff=44.44.44.44 cpt=443 src=12.12.12.12 ver=TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 end=1566300670892 additionalReqHeaders=[{"Accept":"*/*"},{"x-v":"1"},{"x-fapi-interaction-id":"10.10.10.10"}] additionalResHeaders=[{"Content-Type":"text/html; charset\=UTF-8"}] + +CEF:0|Kaspersky Lab|Kaspersky ICAP Server|%VERSION%|%EVENT_CLASS_ID%|%EVENT_NAME%|%SEVERITY%| msg=%EVENT_MSG% src=%CLIENT_IP% dvcpid=%ICAP_SERVER_PID% cs2=%HTTP_USER_NAME% cs2Label=X-Client-Username cs3=%HTTP_USER_IP% cs3Label=X-Client-IP start=%EVENT_TIME% fileHash=%SCANNED_FILE_HASH% request=%SCANNED_URL% cs1=%SCAN_RESULT% cs1Label=Scan result cs4=%VIRUS_NAME% cs4Label=Virus name cs5=%SCANNED_FILE_SHA256_HASH% cs5Label=SHA256 + +<189>1 2021-06-18T10:55:50.000003Z host app - - - CEF:0|Elastic|Vaporware|1.0.0-alpha|18|Web request|low|eventId=3457 msg=hello + +Dec 03 2017 16:31:28.861 IST 10.17.4.208 CEF:0|Aruba Networks|ClearPass|6.5.0.69058|0-1-0|Insight Logs|0|Auth.Username=host/Asif-Test-PC2 Auth.Authorization-Sources=null Auth.Login-Status=216 Auth.Request-Timestamp=2017-12-03 16:28:20+05:30 Auth.Protocol=RADIUS Auth.Source=null Auth.Enforcement-Profiles=[Allow Access Profile] Auth.NAS-Port=null Auth.SSID=cppm-dot1x-test TimestampFormat=MMM dd yyyy HH:mm:ss.SSS zzz Auth.NAS-Port-Type=19 Auth.Error-Code=216 Auth.Roles=null Auth.Service=Test Wireless Auth.Host-MAC-Address=6817294b0636 Auth.Unhealthy=null Auth.NAS-IP-Address=10.17.4.7 src=10.17.4.208 Auth.CalledStationId=000B8661CD70 Auth.NAS-Identifier=ClearPassLab3600 + +unparsable line + +Nov 19 2017 18:22:40.700 IST 10.17.4.221 CEF:0|Aruba Networks|ClearPass|6.5.0.68754|13-1-0|Audit Records|5|cat=Role timeFormat=MMM dd yyyy HH:mm:ss.SSS zzz rt=Nov 19, 2014 18:21:13 IST src=Test Role 10 act=ADD usrName=admin \ No newline at end of file diff --git a/tests/test_cef.py b/tests/test_cef.py new file mode 100644 index 000000000..5154bfffd --- /dev/null +++ b/tests/test_cef.py @@ -0,0 +1,35 @@ +import os +import unittest +import json +import jc.parsers.cef + +THIS_DIR = os.path.dirname(os.path.abspath(__file__)) + + +class MyTests(unittest.TestCase): + + def setUp(self): + # input + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/cef.out'), 'r', encoding='utf-8') as f: + self.cef = f.read() + + # output + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/cef.json'), 'r', encoding='utf-8') as f: + self.cef_json = json.loads(f.read()) + + + def test_cef_nodata(self): + """ + Test 'cef' with no data + """ + self.assertEqual(jc.parsers.cef.parse('', quiet=True), []) + + def test_cef_sample(self): + """ + Test with sample cef log + """ + self.assertEqual(jc.parsers.cef.parse(self.cef, quiet=True), self.cef_json) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_cef_s.py b/tests/test_cef_s.py new file mode 100644 index 000000000..426d2523a --- /dev/null +++ b/tests/test_cef_s.py @@ -0,0 +1,37 @@ +import os +import json +import unittest +import jc.parsers.cef_s + +THIS_DIR = os.path.dirname(os.path.abspath(__file__)) + +# To create streaming output use: +# $ cat cef.out | jc --cef-s | jello -c > cef-streaming.json + + +class MyTests(unittest.TestCase): + + def setUp(self): + # input + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/cef.out'), 'r', encoding='utf-8') as f: + self.cef = f.read() + + # output + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/cef-streaming.json'), 'r', encoding='utf-8') as f: + self.cef_streaming_json = json.loads(f.read()) + + def test_cef_s_nodata(self): + """ + Test 'cef' with no data + """ + self.assertEqual(list(jc.parsers.cef_s.parse([], quiet=True)), []) + + def test_cef_s_sample(self): + """ + Test with sample cef log + """ + self.assertEqual(list(jc.parsers.cef_s.parse(self.cef.splitlines(), quiet=True)), self.cef_streaming_json) + + +if __name__ == '__main__': + unittest.main() From 96c5ef5fa2ed7ec65eb289100de1ceffe952035b Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sat, 20 Aug 2022 13:43:47 -0700 Subject: [PATCH 112/116] add library credit --- jc/parsers/cef_s.py | 1 + 1 file changed, 1 insertion(+) diff --git a/jc/parsers/cef_s.py b/jc/parsers/cef_s.py index 88826689a..6a945e25d 100644 --- a/jc/parsers/cef_s.py +++ b/jc/parsers/cef_s.py @@ -101,6 +101,7 @@ class info(): description = 'CEF string streaming parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' + details = 'Using the pycef library at https://github.com/DavidJBianco/pycef/releases/tag/v1.11-2' compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd'] streaming = True From 65cf7960bf039c400c0fda2eeaa2950083b61b32 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sun, 21 Aug 2022 11:08:10 -0700 Subject: [PATCH 113/116] add traceroute -n tests --- tests/fixtures/generic/traceroute-n-ipv4.json | 1 + tests/fixtures/generic/traceroute-n-ipv4.out | 9 +++++ tests/fixtures/generic/traceroute-n-ipv6.json | 1 + tests/fixtures/generic/traceroute-n-ipv6.out | 13 +++++++ .../generic/traceroute-n-q1-ipv4.json | 1 + .../fixtures/generic/traceroute-n-q1-ipv4.out | 9 +++++ tests/test_traceroute.py | 38 +++++++++++++++++++ 7 files changed, 72 insertions(+) create mode 100644 tests/fixtures/generic/traceroute-n-ipv4.json create mode 100644 tests/fixtures/generic/traceroute-n-ipv4.out create mode 100644 tests/fixtures/generic/traceroute-n-ipv6.json create mode 100644 tests/fixtures/generic/traceroute-n-ipv6.out create mode 100644 tests/fixtures/generic/traceroute-n-q1-ipv4.json create mode 100644 tests/fixtures/generic/traceroute-n-q1-ipv4.out diff --git a/tests/fixtures/generic/traceroute-n-ipv4.json b/tests/fixtures/generic/traceroute-n-ipv4.json new file mode 100644 index 000000000..ddffd6f61 --- /dev/null +++ b/tests/fixtures/generic/traceroute-n-ipv4.json @@ -0,0 +1 @@ +{"destination_ip":"199.58.80.40","destination_name":"www.koumbit.org","hops":[{"hop":1,"probes":[{"annotation":null,"asn":null,"ip":"192.168.2.1","name":null,"rtt":0.967},{"annotation":null,"asn":null,"ip":"192.168.2.1","name":null,"rtt":1.022},{"annotation":null,"asn":null,"ip":"192.168.2.1","name":null,"rtt":1.204}]},{"hop":2,"probes":[{"annotation":null,"asn":null,"ip":"24.212.242.17","name":null,"rtt":10.176},{"annotation":null,"asn":null,"ip":"24.212.242.17","name":null,"rtt":18.136},{"annotation":null,"asn":null,"ip":"24.212.242.17","name":null,"rtt":18.244}]},{"hop":3,"probes":[{"annotation":null,"asn":null,"ip":"10.170.192.58","name":null,"rtt":19.396},{"annotation":null,"asn":null,"ip":"10.170.192.58","name":null,"rtt":19.575},{"annotation":null,"asn":null,"ip":"10.170.192.58","name":null,"rtt":19.572}]},{"hop":4,"probes":[{"annotation":null,"asn":null,"ip":"192.171.61.61","name":null,"rtt":23.072},{"annotation":null,"asn":null,"ip":"192.171.61.61","name":null,"rtt":17.073},{"annotation":null,"asn":null,"ip":"192.171.61.61","name":null,"rtt":23.308}]},{"hop":5,"probes":[{"annotation":null,"asn":null,"ip":"206.248.189.97","name":null,"rtt":20.521},{"annotation":null,"asn":null,"ip":"206.248.189.97","name":null,"rtt":22.837},{"annotation":null,"asn":null,"ip":"206.248.189.97","name":null,"rtt":23.194}]},{"hop":6,"probes":[{"annotation":null,"asn":null,"ip":"198.179.18.41","name":null,"rtt":18.334},{"annotation":null,"asn":null,"ip":"198.179.18.41","name":null,"rtt":17.894},{"annotation":null,"asn":null,"ip":"198.179.18.41","name":null,"rtt":17.792}]},{"hop":7,"probes":[{"annotation":null,"asn":null,"ip":"64.15.69.54","name":null,"rtt":17.056},{"annotation":null,"asn":null,"ip":"64.15.69.54","name":null,"rtt":14.033},{"annotation":null,"asn":null,"ip":"64.15.69.54","name":null,"rtt":12.351}]},{"hop":8,"probes":[{"annotation":null,"asn":null,"ip":"199.58.80.40","name":null,"rtt":18.203},{"annotation":null,"asn":null,"ip":"199.58.80.40","name":null,"rtt":18.789},{"annotation":null,"asn":null,"ip":"199.58.80.40","name":null,"rtt":18.906}]}]} diff --git a/tests/fixtures/generic/traceroute-n-ipv4.out b/tests/fixtures/generic/traceroute-n-ipv4.out new file mode 100644 index 000000000..bf9cfd03b --- /dev/null +++ b/tests/fixtures/generic/traceroute-n-ipv4.out @@ -0,0 +1,9 @@ +traceroute to www.koumbit.org (199.58.80.40), 30 hops max, 60 byte packets + 1 192.168.2.1 0.967 ms 1.022 ms 1.204 ms + 2 24.212.242.17 10.176 ms 18.136 ms 18.244 ms + 3 10.170.192.58 19.396 ms 19.575 ms 19.572 ms + 4 192.171.61.61 23.072 ms 206.248.155.109 17.073 ms 192.171.63.17 23.308 ms + 5 206.248.189.97 20.521 ms 22.837 ms 23.194 ms + 6 198.179.18.41 18.334 ms 17.894 ms 17.792 ms + 7 64.15.69.54 17.056 ms 14.033 ms 12.351 ms + 8 199.58.80.40 18.203 ms 18.789 ms 18.906 ms diff --git a/tests/fixtures/generic/traceroute-n-ipv6.json b/tests/fixtures/generic/traceroute-n-ipv6.json new file mode 100644 index 000000000..7799d705a --- /dev/null +++ b/tests/fixtures/generic/traceroute-n-ipv6.json @@ -0,0 +1 @@ +{"destination_ip":"2607:f8b0:4020:806::2004","destination_name":"www.google.com","hops":[{"hop":1,"probes":[{"annotation":null,"asn":null,"ip":"2605:9000:402:6a01::1","name":null,"rtt":4.181},{"annotation":null,"asn":null,"ip":"2605:9000:402:6a01::1","name":null,"rtt":4.294},{"annotation":null,"asn":null,"ip":"2605:9000:402:6a01::1","name":null,"rtt":4.253}]},{"hop":2,"probes":[{"annotation":null,"asn":null,"ip":"2605:9000:0:400a::f1","name":null,"rtt":0.354},{"annotation":null,"asn":null,"ip":"2605:9000:0:400a::f1","name":null,"rtt":0.532},{"annotation":null,"asn":null,"ip":"2605:9000:0:400a::f1","name":null,"rtt":0.484}]},{"hop":3,"probes":[{"annotation":null,"asn":null,"ip":"2001:5a0:40:100::51","name":null,"rtt":15.284},{"annotation":null,"asn":null,"ip":"2001:5a0:40:100::51","name":null,"rtt":4.864},{"annotation":null,"asn":null,"ip":"2001:5a0:40:100::51","name":null,"rtt":15.415}]},{"hop":4,"probes":[{"annotation":null,"asn":null,"ip":"2001:5a0:40:100::51","name":null,"rtt":15.379},{"annotation":null,"asn":null,"ip":"2001:5a0:40:100::51","name":null,"rtt":12.709},{"annotation":null,"asn":null,"ip":"2001:5a0:40:100::51","name":null,"rtt":15.289}]},{"hop":5,"probes":[{"annotation":null,"asn":null,"ip":"2001:5a0:1900:100::12","name":null,"rtt":10.02},{"annotation":null,"asn":null,"ip":"2001:5a0:1900:100::12","name":null,"rtt":10.212},{"annotation":null,"asn":null,"ip":"2001:5a0:1900:100::12","name":null,"rtt":10.163}]},{"hop":6,"probes":[{"annotation":null,"asn":null,"ip":"2001:5a0:1900:100::12","name":null,"rtt":10.113},{"annotation":null,"asn":null,"ip":"2001:5a0:1900:100::12","name":null,"rtt":8.399},{"annotation":null,"asn":null,"ip":"2001:5a0:1900:100::12","name":null,"rtt":10.215}]},{"hop":7,"probes":[{"annotation":null,"asn":null,"ip":"2001:4860:0:1127::2","name":null,"rtt":9.11},{"annotation":null,"asn":null,"ip":"2001:4860:0:1127::2","name":null,"rtt":8.476},{"annotation":null,"asn":null,"ip":"2001:4860:0:1127::2","name":null,"rtt":8.38}]},{"hop":8,"probes":[{"annotation":null,"asn":null,"ip":"2001:4860::8:4000:cd80","name":null,"rtt":9.428},{"annotation":null,"asn":null,"ip":"2001:4860::8:4000:cd80","name":null,"rtt":9.36},{"annotation":null,"asn":null,"ip":"2001:4860::8:4000:cd80","name":null,"rtt":9.229}]},{"hop":9,"probes":[{"annotation":null,"asn":null,"ip":"2001:4860::9:4001:d508","name":null,"rtt":9.376},{"annotation":null,"asn":null,"ip":"2001:4860::9:4001:d508","name":null,"rtt":9.105},{"annotation":null,"asn":null,"ip":"2001:4860::9:4001:d508","name":null,"rtt":9.384}]},{"hop":10,"probes":[{"annotation":null,"asn":null,"ip":"2001:4860:0:11da::1","name":null,"rtt":8.489},{"annotation":null,"asn":null,"ip":"2001:4860:0:11da::1","name":null,"rtt":8.978},{"annotation":null,"asn":null,"ip":"2001:4860:0:11da::1","name":null,"rtt":9.64}]},{"hop":11,"probes":[{"annotation":null,"asn":null,"ip":"2001:4860:0:1::c73","name":null,"rtt":9.596},{"annotation":null,"asn":null,"ip":"2001:4860:0:1::c73","name":null,"rtt":9.077},{"annotation":null,"asn":null,"ip":"2001:4860:0:1::c73","name":null,"rtt":9.724}]},{"hop":12,"probes":[{"annotation":null,"asn":null,"ip":"2607:f8b0:4020:806::2004","name":null,"rtt":8.086},{"annotation":null,"asn":null,"ip":"2607:f8b0:4020:806::2004","name":null,"rtt":8.091},{"annotation":null,"asn":null,"ip":"2607:f8b0:4020:806::2004","name":null,"rtt":8.436}]}]} diff --git a/tests/fixtures/generic/traceroute-n-ipv6.out b/tests/fixtures/generic/traceroute-n-ipv6.out new file mode 100644 index 000000000..eaa43909f --- /dev/null +++ b/tests/fixtures/generic/traceroute-n-ipv6.out @@ -0,0 +1,13 @@ +traceroute to www.google.com (2607:f8b0:4020:806::2004), 30 hops max, 80 byte packets + 1 2605:9000:402:6a01::1 4.181 ms 4.294 ms 4.253 ms + 2 2605:9000:0:400a::f1 0.354 ms 0.532 ms 0.484 ms + 3 2001:5a0:40:100::51 15.284 ms 2605:9000:0:101::1 4.864 ms 2001:5a0:40:100::51 15.415 ms + 4 2001:5a0:40:100::51 15.379 ms 2001:5a0:300:200::202 12.709 ms 2001:5a0:40:100::51 15.289 ms + 5 2001:5a0:1900:100::12 10.020 ms 10.212 ms 10.163 ms + 6 2001:5a0:1900:100::12 10.113 ms 2001:5a0:400:700::17 8.399 ms 2001:5a0:1900:100::12 10.215 ms + 7 2001:4860:0:1127::2 9.110 ms 2001:5a0:400:700::17 8.476 ms 8.380 ms + 8 2001:4860::8:4000:cd80 9.428 ms 2001:4860:0:1128::14 9.360 ms 2001:4860::8:4000:cd80 9.229 ms + 9 2001:4860::9:4001:d508 9.376 ms 2001:4860::c:4002:652a 9.105 ms 2001:4860::c:4002:6523 9.384 ms +10 2001:4860:0:11da::1 8.489 ms 2001:4860::9:4001:d508 8.978 ms 2001:4860::1c:4000:f5eb 9.640 ms +11 2001:4860:0:1::c73 9.596 ms 9.077 ms 9.724 ms +12 2607:f8b0:4020:806::2004 8.086 ms 8.091 ms 8.436 ms diff --git a/tests/fixtures/generic/traceroute-n-q1-ipv4.json b/tests/fixtures/generic/traceroute-n-q1-ipv4.json new file mode 100644 index 000000000..450f3f27a --- /dev/null +++ b/tests/fixtures/generic/traceroute-n-q1-ipv4.json @@ -0,0 +1 @@ +{"destination_ip":"199.58.80.40","destination_name":"www.koumbit.org","hops":[{"hop":1,"probes":[{"annotation":null,"asn":null,"ip":"192.168.2.1","name":null,"rtt":3.425}]},{"hop":2,"probes":[{"annotation":null,"asn":null,"ip":"24.212.242.17","name":null,"rtt":16.153}]},{"hop":3,"probes":[{"annotation":null,"asn":null,"ip":"10.170.192.58","name":null,"rtt":17.231}]},{"hop":4,"probes":[{"annotation":null,"asn":null,"ip":"192.171.61.161","name":null,"rtt":25.393}]},{"hop":5,"probes":[{"annotation":null,"asn":null,"ip":"206.248.189.97","name":null,"rtt":25.322}]},{"hop":6,"probes":[{"annotation":null,"asn":null,"ip":"198.179.18.41","name":null,"rtt":23.755}]},{"hop":7,"probes":[{"annotation":null,"asn":null,"ip":"64.15.69.54","name":null,"rtt":25.091}]},{"hop":8,"probes":[{"annotation":null,"asn":null,"ip":"199.58.80.40","name":null,"rtt":25.196}]}]} diff --git a/tests/fixtures/generic/traceroute-n-q1-ipv4.out b/tests/fixtures/generic/traceroute-n-q1-ipv4.out new file mode 100644 index 000000000..efa0f6340 --- /dev/null +++ b/tests/fixtures/generic/traceroute-n-q1-ipv4.out @@ -0,0 +1,9 @@ +traceroute to www.koumbit.org (199.58.80.40), 30 hops max, 60 byte packets + 1 192.168.2.1 3.425 ms + 2 24.212.242.17 16.153 ms + 3 10.170.192.58 17.231 ms + 4 192.171.61.161 25.393 ms + 5 206.248.189.97 25.322 ms + 6 198.179.18.41 23.755 ms + 7 64.15.69.54 25.091 ms + 8 199.58.80.40 25.196 ms diff --git a/tests/test_traceroute.py b/tests/test_traceroute.py index bb0872e6c..5c68fc48d 100644 --- a/tests/test_traceroute.py +++ b/tests/test_traceroute.py @@ -64,6 +64,16 @@ def setUp(self): with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/traceroute8.out'), 'r', encoding='utf-8') as f: self.generic_traceroute8 = f.read() + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/traceroute-n-ipv4.out'), 'r', encoding='utf-8') as f: + self.generic_traceroute_n_ipv4 = f.read() + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/traceroute-n-q1-ipv4.out'), 'r', encoding='utf-8') as f: + self.generic_traceroute_n_q1_ipv4 = f.read() + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/traceroute-n-ipv6.out'), 'r', encoding='utf-8') as f: + self.generic_traceroute_n_ipv6 = f.read() + + # output with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/osx-10.14.6/traceroute-no-header.json'), 'r', encoding='utf-8') as f: self.osx_10_14_6_traceroute_no_header_json = json.loads(f.read()) @@ -119,6 +129,16 @@ def setUp(self): with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/traceroute8.json'), 'r', encoding='utf-8') as f: self.generic_traceroute8_json = json.loads(f.read()) + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/traceroute-n-ipv4.json'), 'r', encoding='utf-8') as f: + self.generic_traceroute_n_ipv4_json = json.loads(f.read()) + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/traceroute-n-q1-ipv4.json'), 'r', encoding='utf-8') as f: + self.generic_traceroute_n_q1_ipv4_json = json.loads(f.read()) + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/traceroute-n-ipv6.json'), 'r', encoding='utf-8') as f: + self.generic_traceroute_n_ipv6_json = json.loads(f.read()) + + def test_traceroute_nodata(self): """ Test 'traceroute' with no data @@ -233,6 +253,24 @@ def test_traceroute8_generic(self): """ self.assertEqual(jc.parsers.traceroute.parse(self.generic_traceroute8, quiet=True), self.generic_traceroute8_json) + def test_traceroute_n_ipv4(self): + """ + Test 'traceroute -n x.x.x.x' + """ + self.assertEqual(jc.parsers.traceroute.parse(self.generic_traceroute_n_ipv4, quiet=True), self.generic_traceroute_n_ipv4_json) + + def test_traceroute_n_q1_ipv4(self): + """ + Test 'traceroute -q1 -n x.x.x.x' + """ + self.assertEqual(jc.parsers.traceroute.parse(self.generic_traceroute_n_q1_ipv4, quiet=True), self.generic_traceroute_n_q1_ipv4_json) + + def test_traceroute_n_ipv6(self): + """ + Test 'traceroute6 -n x::x' + """ + self.assertEqual(jc.parsers.traceroute.parse(self.generic_traceroute_n_ipv6, quiet=True), self.generic_traceroute_n_ipv6_json) + if __name__ == '__main__': unittest.main() From 4f21c7b7b4b0f815fc340cfa1404a4b249cfc684 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sun, 21 Aug 2022 12:38:55 -0700 Subject: [PATCH 114/116] change --time-out to --meta-out. add more meta fields --- README.md | 2 +- completions/jc_bash_completion.sh | 2 +- completions/jc_zsh_completion.sh | 6 ++-- jc/cli.py | 37 +++++++++++++++++-------- jc/cli_data.py | 2 +- man/jc.1 | 10 +++---- templates/manpage_template | 8 +++--- templates/readme_template | 2 +- tests/{test_cli.py => test_jc_cli.py} | 40 ++++++++++++++++----------- 9 files changed, 65 insertions(+), 44 deletions(-) rename tests/{test_cli.py => test_jc_cli.py} (86%) diff --git a/README.md b/README.md index e4b0f32dd..913521116 100644 --- a/README.md +++ b/README.md @@ -280,10 +280,10 @@ option. | `-d` | `--debug` | Debug mode. Prints trace messages if parsing issues are encountered (use`-dd` for verbose debugging) | | `-h` | `--help` | Help. Use `jc -h --parser_name` for parser documentation | | `-m` | `--monochrome` | Monochrome output | +| `-M` | `--meta-out` | Add metadata to output including timestamp, parser name, magic command, magic command exit code, etc. | | | `-p` | `--pretty` | Pretty format the JSON output | | `-q` | `--quiet` | Quiet mode. Suppresses parser warning messages (use `-qq` to ignore streaming parser errors) | | `-r` | `--raw` | Raw output. Provides more literal output, typically with string values and no additional semantic processing | -| `-t` | `--time-out` | Add UTC Unix timestamp information to output | | `-u` | `--unbuffer` | Unbuffer output | | `-v` | `--version` | Version information | | `-y` | `--yaml-out` | YAML output | diff --git a/completions/jc_bash_completion.sh b/completions/jc_bash_completion.sh index 8b2901d6b..3fa432e01 100644 --- a/completions/jc_bash_completion.sh +++ b/completions/jc_bash_completion.sh @@ -5,7 +5,7 @@ _jc() jc_commands=(acpi airport arp blkid chage cksum crontab date df dig dmidecode dpkg du env file finger free git gpg hciconfig id ifconfig iostat iptables iw jobs last lastb ls lsblk lsmod lsof lsusb md5 md5sum mdadm mount mpstat netstat nmcli ntpq pidstat ping ping6 pip pip3 postconf printenv ps route rpm rsync sfdisk sha1sum sha224sum sha256sum sha384sum sha512sum shasum ss stat sum sysctl systemctl systeminfo timedatectl top tracepath tracepath6 traceroute traceroute6 ufw uname update-alternatives upower uptime vdir vmstat w wc who xrandr zipinfo) jc_parsers=(--acpi --airport --airport-s --arp --asciitable --asciitable-m --blkid --cef --cef-s --chage --cksum --crontab --crontab-u --csv --csv-s --date --df --dig --dir --dmidecode --dpkg-l --du --email-address --env --file --finger --free --fstab --git-log --git-log-s --gpg --group --gshadow --hash --hashsum --hciconfig --history --hosts --id --ifconfig --ini --iostat --iostat-s --ip-address --iptables --iso-datetime --iw-scan --jar-manifest --jobs --jwt --kv --last --ls --ls-s --lsblk --lsmod --lsof --lsusb --m3u --mdadm --mount --mpstat --mpstat-s --netstat --nmcli --ntpq --passwd --pidstat --pidstat-s --ping --ping-s --pip-list --pip-show --plist --postconf --ps --route --rpm-qi --rsync --rsync-s --sfdisk --shadow --ss --stat --stat-s --sysctl --syslog --syslog-s --syslog-bsd --syslog-bsd-s --systemctl --systemctl-lj --systemctl-ls --systemctl-luf --systeminfo --time --timedatectl --timestamp --top --top-s --tracepath --traceroute --ufw --ufw-appinfo --uname --update-alt-gs --update-alt-q --upower --uptime --url --vmstat --vmstat-s --w --wc --who --x509-cert --xml --xrandr --yaml --zipinfo) - jc_options=(--force-color -C --debug -d --monochrome -m --pretty -p --quiet -q --raw -r --time-out -t --unbuffer -u --yaml-out -y) + jc_options=(--force-color -C --debug -d --monochrome -m --meta-out -M --pretty -p --quiet -q --raw -r --unbuffer -u --yaml-out -y) jc_about_options=(--about -a) jc_about_mod_options=(--pretty -p --yaml-out -y --monochrome -m --force-color -C) jc_help_options=(--help -h) diff --git a/completions/jc_zsh_completion.sh b/completions/jc_zsh_completion.sh index d89e4c7c9..ec87a73bf 100644 --- a/completions/jc_zsh_completion.sh +++ b/completions/jc_zsh_completion.sh @@ -218,7 +218,7 @@ _jc() { '--yaml:YAML file parser' '--zipinfo:`zipinfo` command parser' ) - jc_options=(--force-color -C --debug -d --monochrome -m --pretty -p --quiet -q --raw -r --time-out -t --unbuffer -u --yaml-out -y) + jc_options=(--force-color -C --debug -d --monochrome -m --meta-out -M --pretty -p --quiet -q --raw -r --unbuffer -u --yaml-out -y) jc_options_describe=( '--force-color:force color output even when using pipes (overrides -m)' '-C:force color output even when using pipes (overrides -m)' @@ -226,14 +226,14 @@ _jc() { '-d:debug (double for verbose debug)' '--monochrome:monochrome output' '-m:monochrome output' + '--meta-out:add metadata to output including timestamp, etc.' + '-M:add metadata to output including timestamp, etc.' '--pretty:pretty print output' '-p:pretty print output' '--quiet:suppress warnings (double to ignore streaming errors)' '-q:suppress warnings (double to ignore streaming errors)' '--raw:raw output' '-r:raw output' - '--time-out:add UTC Unix timestamp information to output' - '-t:add UTC Unix timestamp information to output' '--unbuffer:unbuffer output' '-u:unbuffer output' '--yaml-out:YAML output' diff --git a/jc/cli.py b/jc/cli.py index cffd4c14a..b3ffff99f 100644 --- a/jc/cli.py +++ b/jc/cli.py @@ -445,28 +445,41 @@ def combined_exit_code(program_exit=0, jc_exit=0): return exit_code -def add_timestamp_to(list_or_dict, runtime, magic_exit_code): +def add_metadata_to(list_or_dict, + runtime=None, + run_command=None, + magic_exit_code=None, + parser_name=None): """ This function mutates a list or dict in place. If the _jc_meta field - does not already exist, it will be created with the timestamp field. If - the _jc_meta field already exists, the timestamp will be added to the - existing object. + does not already exist, it will be created with the metadata fields. If + the _jc_meta field already exists, the metadata fields will be added to + the existing object. """ run_timestamp = runtime.timestamp() - timestamp_obj = {'timestamp': run_timestamp} + + if not run_command: + magic_exit_code = None + + meta_obj = { + 'parser': parser_name, + 'magic_command': run_command, + 'magic_command_exit': magic_exit_code, + 'timestamp': run_timestamp + } if isinstance(list_or_dict, dict): if '_jc_meta' not in list_or_dict: list_or_dict['_jc_meta'] = {} - list_or_dict['_jc_meta'].update(timestamp_obj) + list_or_dict['_jc_meta'].update(meta_obj) elif isinstance(list_or_dict, list): for item in list_or_dict: if '_jc_meta' not in item: item['_jc_meta'] = {} - item['_jc_meta'].update(timestamp_obj) + item['_jc_meta'].update(meta_obj) else: utils.error_message(['Parser returned an unsupported object type.']) @@ -517,7 +530,7 @@ def main(): quiet = 'q' in options ignore_exceptions = options.count('q') > 1 raw = 'r' in options - timestamp = 't' in options + meta_out = 'M' in options unbuffer = 'u' in options version_info = 'v' in options yaml_out = 'y' in options @@ -632,9 +645,9 @@ def main(): ignore_exceptions=ignore_exceptions) for line in result: - if timestamp: + if meta_out: run_dt_utc = datetime.now(timezone.utc) - add_timestamp_to(line, run_dt_utc, magic_exit_code) + add_metadata_to(line, run_dt_utc, run_command, magic_exit_code, parser_name) safe_print_out(line, pretty=pretty, @@ -661,9 +674,9 @@ def main(): raw=raw, quiet=quiet) - if timestamp: + if meta_out: run_dt_utc = datetime.now(timezone.utc) - add_timestamp_to(result, run_dt_utc, magic_exit_code) + add_metadata_to(result, run_dt_utc, run_command, magic_exit_code, parser_name) safe_print_out(result, pretty=pretty, diff --git a/jc/cli_data.py b/jc/cli_data.py index 489f2126c..7147e8972 100644 --- a/jc/cli_data.py +++ b/jc/cli_data.py @@ -7,10 +7,10 @@ '--debug': ['d', 'debug (double for verbose debug)'], '--help': ['h', 'help (--help --parser_name for parser documentation)'], '--monochrome': ['m', 'monochrome output'], + '--meta-out': ['M', 'add metadata to output including timestamp, etc.'], '--pretty': ['p', 'pretty print output'], '--quiet': ['q', 'suppress warnings (double to ignore streaming errors)'], '--raw': ['r', 'raw output'], - '--time-out': ['t', 'add UTC Unix timestamp information to output'], '--unbuffer': ['u', 'unbuffer output'], '--version': ['v', 'version info'], '--yaml-out': ['y', 'YAML output'], diff --git a/man/jc.1 b/man/jc.1 index 002d11a08..2e9f17112 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -1,4 +1,4 @@ -.TH jc 1 2022-08-20 1.21.0 "JSON Convert" +.TH jc 1 2022-08-21 1.21.0 "JSON Convert" .SH NAME \fBjc\fP \- JSON Convert JSONifies the output of many CLI tools and file-types .SH SYNOPSIS @@ -646,6 +646,10 @@ Help (\fB--help --parser_name\fP for parser documentation) Monochrome output .TP .B +\fB-M\fP, \fB--meta-out\fP +Add metadata to output including timestamp, parser name, magic command, magic command exit code, etc. +.TP +.B \fB-p\fP, \fB--pretty\fP Pretty print output .TP @@ -658,10 +662,6 @@ Quiet mode. Suppresses parser warning messages (use -qq to ignore streaming pars Raw output. Provides more literal output, typically with string values and no additional semantic processing .TP .B -\fB-t\fP, \fB--time-out\fP -Add UTC Unix timestamp information to output -.TP -.B \fB-u\fP, \fB--unbuffer\fP Unbuffer output (useful for slow streaming data with streaming parsers) .TP diff --git a/templates/manpage_template b/templates/manpage_template index 26a973070..7c6c7d332 100644 --- a/templates/manpage_template +++ b/templates/manpage_template @@ -51,6 +51,10 @@ Help (\fB--help --parser_name\fP for parser documentation) Monochrome output .TP .B +\fB-M\fP, \fB--meta-out\fP +Add metadata to output including timestamp, parser name, magic command, magic command exit code, etc. +.TP +.B \fB-p\fP, \fB--pretty\fP Pretty print output .TP @@ -63,10 +67,6 @@ Quiet mode. Suppresses parser warning messages (use -qq to ignore streaming pars Raw output. Provides more literal output, typically with string values and no additional semantic processing .TP .B -\fB-t\fP, \fB--time-out\fP -Add UTC Unix timestamp information to output -.TP -.B \fB-u\fP, \fB--unbuffer\fP Unbuffer output (useful for slow streaming data with streaming parsers) .TP diff --git a/templates/readme_template b/templates/readme_template index 32c5a0aaa..c4e64d897 100644 --- a/templates/readme_template +++ b/templates/readme_template @@ -161,10 +161,10 @@ option. | `-d` | `--debug` | Debug mode. Prints trace messages if parsing issues are encountered (use`-dd` for verbose debugging) | | `-h` | `--help` | Help. Use `jc -h --parser_name` for parser documentation | | `-m` | `--monochrome` | Monochrome output | +| `-M` | `--meta-out` | Add metadata to output including timestamp, parser name, magic command, magic command exit code, etc. | | | `-p` | `--pretty` | Pretty format the JSON output | | `-q` | `--quiet` | Quiet mode. Suppresses parser warning messages (use `-qq` to ignore streaming parser errors) | | `-r` | `--raw` | Raw output. Provides more literal output, typically with string values and no additional semantic processing | -| `-t` | `--time-out` | Add UTC Unix timestamp information to output | | `-u` | `--unbuffer` | Unbuffer output | | `-v` | `--version` | Version information | | `-y` | `--yaml-out` | YAML output | diff --git a/tests/test_cli.py b/tests/test_jc_cli.py similarity index 86% rename from tests/test_cli.py rename to tests/test_jc_cli.py index 3368f346f..48a642b31 100644 --- a/tests/test_cli.py +++ b/tests/test_jc_cli.py @@ -255,39 +255,47 @@ def test_cli_about_jc(self): self.assertGreaterEqual(jc.cli.about_jc()['parser_count'], 55) self.assertEqual(jc.cli.about_jc()['parser_count'], len(jc.cli.about_jc()['parsers'])) - def test_add_timestamp_to_simple_dict(self): + def test_add_meta_to_simple_dict(self): list_or_dict = {'a': 1, 'b': 2} runtime = datetime(2022, 8, 5, 0, 37, 9, 273349, tzinfo=timezone.utc) - magic_exit_code = 0 - expected = {'a': 1, 'b': 2, '_jc_meta': {'timestamp': 1659659829.273349}} - jc.cli.add_timestamp_to(list_or_dict, runtime, magic_exit_code) + magic_exit_code = 2 + run_command = ['ping', '-c3', '192.168.1.123'] + parser_name = 'ping' + expected = {'a': 1, 'b': 2, '_jc_meta': {'parser': 'ping', 'magic_command': ['ping', '-c3', '192.168.1.123'], 'magic_command_exit': 2, 'timestamp': 1659659829.273349}} + jc.cli.add_metadata_to(list_or_dict, runtime, run_command, magic_exit_code, parser_name) self.assertEqual(list_or_dict, expected) - def test_add_timestamp_to_simple_list(self): + def test_add_meta_to_simple_list(self): list_or_dict = [{'a': 1, 'b': 2},{'a': 3, 'b': 4}] runtime = datetime(2022, 8, 5, 0, 37, 9, 273349, tzinfo=timezone.utc) - magic_exit_code = 0 - expected = [{'a': 1, 'b': 2, '_jc_meta': {'timestamp': 1659659829.273349}}, {'a': 3, 'b': 4, '_jc_meta': {'timestamp': 1659659829.273349}}] - jc.cli.add_timestamp_to(list_or_dict, runtime, magic_exit_code) + magic_exit_code = 2 + run_command = ['ping', '-c3', '192.168.1.123'] + parser_name = 'ping' + expected = [{'a': 1, 'b': 2, '_jc_meta': {'parser': 'ping', 'magic_command': ['ping', '-c3', '192.168.1.123'], 'magic_command_exit': 2, 'timestamp': 1659659829.273349}}, {'a': 3, 'b': 4, '_jc_meta': {'parser': 'ping', 'magic_command': ['ping', '-c3', '192.168.1.123'], 'magic_command_exit': 2, 'timestamp': 1659659829.273349}}] + jc.cli.add_metadata_to(list_or_dict, runtime, run_command, magic_exit_code, parser_name) self.assertEqual(list_or_dict, expected) - def test_add_timestamp_to_dict_existing_meta(self): + def test_add_meta_to_dict_existing_meta(self): list_or_dict = {'a': 1, 'b': 2, '_jc_meta': {'foo': 'bar'}} runtime = datetime(2022, 8, 5, 0, 37, 9, 273349, tzinfo=timezone.utc) - magic_exit_code = 0 - expected = {'a': 1, 'b': 2, '_jc_meta': {'foo': 'bar', 'timestamp': 1659659829.273349}} - jc.cli.add_timestamp_to(list_or_dict, runtime, magic_exit_code) + magic_exit_code = 2 + run_command = ['ping', '-c3', '192.168.1.123'] + parser_name = 'ping' + expected = {'a': 1, 'b': 2, '_jc_meta': {'foo': 'bar', 'parser': 'ping', 'magic_command': ['ping', '-c3', '192.168.1.123'], 'magic_command_exit': 2, 'timestamp': 1659659829.273349}} + jc.cli.add_metadata_to(list_or_dict, runtime, run_command, magic_exit_code, parser_name) self.assertEqual(list_or_dict, expected) - def test_add_timestamp_to_list_existing_meta(self): + def test_add_meta_to_list_existing_meta(self): list_or_dict = [{'a': 1, 'b': 2, '_jc_meta': {'foo': 'bar'}},{'a': 3, 'b': 4, '_jc_meta': {'foo': 'bar'}}] runtime = datetime(2022, 8, 5, 0, 37, 9, 273349, tzinfo=timezone.utc) - magic_exit_code = 0 - expected = [{'a': 1, 'b': 2, '_jc_meta': {'foo': 'bar', 'timestamp': 1659659829.273349}}, {'a': 3, 'b': 4, '_jc_meta': {'foo': 'bar', 'timestamp': 1659659829.273349}}] - jc.cli.add_timestamp_to(list_or_dict, runtime, magic_exit_code) + magic_exit_code = 2 + run_command = ['ping', '-c3', '192.168.1.123'] + parser_name = 'ping' + expected = [{'a': 1, 'b': 2, '_jc_meta': {'foo': 'bar', 'parser': 'ping', 'magic_command': ['ping', '-c3', '192.168.1.123'], 'magic_command_exit': 2, 'timestamp': 1659659829.273349}}, {'a': 3, 'b': 4, '_jc_meta': {'foo': 'bar', 'parser': 'ping', 'magic_command': ['ping', '-c3', '192.168.1.123'], 'magic_command_exit': 2, 'timestamp': 1659659829.273349}}] + jc.cli.add_metadata_to(list_or_dict, runtime, run_command, magic_exit_code, parser_name) self.assertEqual(list_or_dict, expected) From 13bba1f4b8fced68acaaf50ca19a08b076f5f15f Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sun, 21 Aug 2022 13:52:39 -0700 Subject: [PATCH 115/116] only show magic command fields if magic as used --- jc/cli.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/jc/cli.py b/jc/cli.py index b3ffff99f..a53ad99df 100644 --- a/jc/cli.py +++ b/jc/cli.py @@ -458,16 +458,15 @@ def add_metadata_to(list_or_dict, """ run_timestamp = runtime.timestamp() - if not run_command: - magic_exit_code = None - meta_obj = { 'parser': parser_name, - 'magic_command': run_command, - 'magic_command_exit': magic_exit_code, 'timestamp': run_timestamp } + if run_command: + meta_obj['magic_command'] = run_command + meta_obj['magic_command_exit'] = magic_exit_code + if isinstance(list_or_dict, dict): if '_jc_meta' not in list_or_dict: list_or_dict['_jc_meta'] = {} From cd2f139409d1901d13f3a91e2a0f6240aa1782f4 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sun, 21 Aug 2022 13:52:52 -0700 Subject: [PATCH 116/116] doc update --- CHANGELOG | 9 ++--- EXAMPLES.md | 82 +++++++++++++++++++++++++++++++++++++++++++++ docs/parsers/cef.md | 2 +- jc/parsers/cef.py | 2 +- 4 files changed, 89 insertions(+), 6 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 6316bb44a..06ae27f27 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,13 +1,14 @@ jc changelog -xxxxxxxx v1.21.0 +20220821 v1.21.0 - Add IP Address string parser -- Add Syslog string parsers (RFC 3164 and RFC 5424) -- Add CEF string parser +- Add Syslog standard and streaming string parsers (RFC 3164 and RFC 5424) +- Add CEF standard and streaming string parser - Add PLIST file parser (XML and binary support) - Add `-n` support to the `traceroute` parser - Add `mdadm` command parser tested on linux -- Add `--time-out` or `-t` option to add a UTC timestamp to the JSON output +- Add `--meta-out` or `-M` option to add metadata to the JSON output, including + a UTC timestamp, parser name, magic command, and magic command exit code - Fix `lsusb` command parser for output containing a `Device Qualifier` and `Binary Object Store Descriptor` sections - Change `LANG=C` to `LC_ALL=C` in locale instructions diff --git a/EXAMPLES.md b/EXAMPLES.md index b7e1a096a..2019bc7cb 100644 --- a/EXAMPLES.md +++ b/EXAMPLES.md @@ -265,6 +265,37 @@ blkid -o udev -ip /dev/sda2 | jc --blkid -p # or: jc -p blkid -o udev } ] ``` +### CEF strings +```bash +cat cef.log | jc --cef -p +``` +```json +[ + { + "deviceVendor": "Trend Micro", + "deviceProduct": "Deep Security Agent", + "deviceVersion": "", + "deviceEventClassId": "4000000", + "name": "Eicar_test_file", + "agentSeverity": 6, + "CEFVersion": 0, + "dvchost": "hostname", + "string": "hello \"world\"!", + "start": "Nov 08 2020 12:30:00.111 UTC", + "start_epoch": 1604867400, + "start_epoch_utc": 1604838600, + "Host_ID": 1, + "Quarantine": 205, + "myDate": "Nov 08 2022 12:30:00.111", + "myDate_epoch": 1667939400, + "myDate_epoch_utc": null, + "myFloat": 3.14, + "deviceEventClassIdNum": 4000000, + "agentSeverityString": "Medium", + "agentSeverityNum": 6 + } +] +``` ### chage --list ```bash chage --list joeuser | jc --chage -p # or: jc -p chage --list joeuser @@ -3466,6 +3497,57 @@ sysctl -a | jc --sysctl -p # or: jc -p sysctl -a "user.expr_nest_max": 32 } ``` +### Syslog strings (RFC 5424) +```bash +cat syslog.txt | jc --syslog -p +``` +```json +[ + { + "priority": 35, + "version": 1, + "timestamp": "2003-10-11T22:14:15.003Z", + "hostname": "mymachine.example.com", + "appname": "evntslog", + "proc_id": null, + "msg_id": "ID47", + "structured_data": [ + { + "identity": "exampleSDID@32473", + "parameters": { + "iut": "3", + "eventSource": "Application", + "eventID": "1011" + } + }, + { + "identity": "examplePriority@32473", + "parameters": { + "class": "high" + } + } + ], + "message": "unauthorized attempt", + "timestamp_epoch": 1065935655, + "timestamp_epoch_utc": 1065910455 + } +] +``` +### Syslog strings (RFC 3164) +```bash +cat syslog.txt | jc --syslog-bsd -p +``` +```json +[ + { + "priority": 34, + "date": "Oct 11 22:14:15", + "hostname": "mymachine", + "tag": "su", + "content": "'su root' failed for lonvick on /dev/pts/8" + } +] +``` ### systemctl ```bash systemctl -a | jc --systemctl -p # or: jc -p systemctl -a diff --git a/docs/parsers/cef.md b/docs/parsers/cef.md index c264cc544..f6ee4b607 100644 --- a/docs/parsers/cef.md +++ b/docs/parsers/cef.md @@ -90,7 +90,7 @@ Examples: "myDate_epoch": 1667939400, "myDate_epoch_utc": null, "myFloat": 3.14, - "deviceEventClassIdNum": 4000000 + "deviceEventClassIdNum": 4000000, "agentSeverityString": "Medium", "agentSeverityNum": 6 } diff --git a/jc/parsers/cef.py b/jc/parsers/cef.py index 44d4f1c5a..0a754fb15 100644 --- a/jc/parsers/cef.py +++ b/jc/parsers/cef.py @@ -85,7 +85,7 @@ "myDate_epoch": 1667939400, "myDate_epoch_utc": null, "myFloat": 3.14, - "deviceEventClassIdNum": 4000000 + "deviceEventClassIdNum": 4000000, "agentSeverityString": "Medium", "agentSeverityNum": 6 }