Merge pull request #273 from kellyjonbrazil/dev

Dev v1.21.0
kellyjonbrazil · Aug 21, 2022 · 37835c1 · 37835c1
2 parents e2f1b16 + cd2f139
commit 37835c1
Show file tree

Hide file tree

Showing 305 changed files with 9,788 additions and 247 deletions.
diff --git a/CHANGELOG b/CHANGELOG
@@ -1,5 +1,20 @@
 jc changelog
 
+20220821 v1.21.0
+- Add IP Address string parser
+- Add Syslog standard and streaming string parsers (RFC 3164 and RFC 5424)
+- Add CEF standard and streaming string parser
+- Add PLIST file parser (XML and binary support)
+- Add `-n` support to the `traceroute` parser
+- Add `mdadm` command parser tested on linux
+- Add `--meta-out` or `-M` option to add metadata to the JSON output, including
+  a UTC timestamp, parser name, magic command, and magic command exit code
+- Fix `lsusb` command parser for output containing a `Device Qualifier` and
+  `Binary Object Store Descriptor` sections
+- Change `LANG=C` to `LC_ALL=C` in locale instructions
+- Add `__main__.py` to package allowing `python -m jc` usage
+- Add an enclosing top-level folder inside the windows.zip package
+
 20220723 v1.20.4
 - Fix URL string parser path list for URLs ending in a forward slash
 

diff --git a/EXAMPLES.md b/EXAMPLES.md
@@ -265,6 +265,37 @@ blkid -o udev -ip /dev/sda2 | jc --blkid -p          # or:  jc -p blkid -o udev
   }
 ]
 ```
+### CEF strings
+```bash
+cat cef.log | jc --cef -p
+```
+```json
+[
+  {
+    "deviceVendor": "Trend Micro",
+    "deviceProduct": "Deep Security Agent",
+    "deviceVersion": "<DSA version>",
+    "deviceEventClassId": "4000000",
+    "name": "Eicar_test_file",
+    "agentSeverity": 6,
+    "CEFVersion": 0,
+    "dvchost": "hostname",
+    "string": "hello \"world\"!",
+    "start": "Nov 08 2020 12:30:00.111 UTC",
+    "start_epoch": 1604867400,
+    "start_epoch_utc": 1604838600,
+    "Host_ID": 1,
+    "Quarantine": 205,
+    "myDate": "Nov 08 2022 12:30:00.111",
+    "myDate_epoch": 1667939400,
+    "myDate_epoch_utc": null,
+    "myFloat": 3.14,
+    "deviceEventClassIdNum": 4000000,
+    "agentSeverityString": "Medium",
+    "agentSeverityNum": 6
+  }
+]
+```
 ### chage --list
 ```bash
 chage --list joeuser | jc --chage -p          # or:  jc -p chage --list joeuser
@@ -1665,6 +1696,65 @@ $ iostat | jc --iostat -p          # or:  jc -p iostat
   }
 ]
 ```
+### IP Address strings
+```bash
+echo 192.168.2.10/24 | jc --ip-address -p
+```
+```json
+{
+  "version": 4,
+  "max_prefix_length": 32,
+  "ip": "192.168.2.10",
+  "ip_compressed": "192.168.2.10",
+  "ip_exploded": "192.168.2.10",
+  "scope_id": null,
+  "ipv4_mapped": null,
+  "six_to_four": null,
+  "teredo_client": null,
+  "teredo_server": null,
+  "dns_ptr": "10.2.168.192.in-addr.arpa",
+  "network": "192.168.2.0",
+  "broadcast": "192.168.2.255",
+  "hostmask": "0.0.0.255",
+  "netmask": "255.255.255.0",
+  "cidr_netmask": 24,
+  "hosts": 254,
+  "first_host": "192.168.2.1",
+  "last_host": "192.168.2.254",
+  "is_multicast": false,
+  "is_private": true,
+  "is_global": false,
+  "is_link_local": false,
+  "is_loopback": false,
+  "is_reserved": false,
+  "is_unspecified": false,
+  "int": {
+    "ip": 3232236042,
+    "network": 3232236032,
+    "broadcast": 3232236287,
+    "first_host": 3232236033,
+    "last_host": 3232236286
+  },
+  "hex": {
+    "ip": "c0:a8:02:0a",
+    "network": "c0:a8:02:00",
+    "broadcast": "c0:a8:02:ff",
+    "hostmask": "00:00:00:ff",
+    "netmask": "ff:ff:ff:00",
+    "first_host": "c0:a8:02:01",
+    "last_host": "c0:a8:02:fe"
+  },
+  "bin": {
+    "ip": "11000000101010000000001000001010",
+    "network": "11000000101010000000001000000000",
+    "broadcast": "11000000101010000000001011111111",
+    "hostmask": "00000000000000000000000011111111",
+    "netmask": "11111111111111111111111100000000",
+    "first_host": "11000000101010000000001000000001",
+    "last_host": "11000000101010000000001011111110"
+  }
+}
+```
 ### iptables
 ```bash
 iptables --line-numbers -v -L -t nat | jc --iptables -p          # or:  jc -p iptables --line-numbers -v -L -t nat
@@ -2832,6 +2922,31 @@ pip show wrapt wheel | jc --pip-show -p          # or:  jc -p pip show wrapt whe
   }
 ]
 ```
+### PLIST files
+```bash
+cat info.plist | jc --plist -p
+```
+```json
+{
+  "NSAppleScriptEnabled": true,
+  "LSMultipleInstancesProhibited": true,
+  "CFBundleInfoDictionaryVersion": "6.0",
+  "DTPlatformVersion": "GM",
+  "CFBundleIconFile": "GarageBand.icns",
+  "CFBundleName": "GarageBand",
+  "DTSDKName": "macosx10.13internal",
+  "NSSupportsAutomaticGraphicsSwitching": true,
+  "RevisionDate": "2018-12-03_14:10:56",
+  "UTImportedTypeDeclarations": [
+    {
+      "UTTypeConformsTo": [
+        "public.data",
+        "public.content"
+      ]
+    }
+  ]
+}
+```
 ### postconf -M
 ```bash
 postconf -M | jc --postconf -p          # or jc -p postconf -M
@@ -3382,6 +3497,57 @@ sysctl -a | jc --sysctl -p          # or:  jc -p sysctl -a
   "user.expr_nest_max": 32
 }
 ```
+### Syslog strings (RFC 5424)
+```bash
+cat syslog.txt | jc --syslog -p
+```
+```json
+[
+  {
+    "priority": 35,
+    "version": 1,
+    "timestamp": "2003-10-11T22:14:15.003Z",
+    "hostname": "mymachine.example.com",
+    "appname": "evntslog",
+    "proc_id": null,
+    "msg_id": "ID47",
+    "structured_data": [
+      {
+        "identity": "exampleSDID@32473",
+        "parameters": {
+          "iut": "3",
+          "eventSource": "Application",
+          "eventID": "1011"
+        }
+      },
+      {
+        "identity": "examplePriority@32473",
+        "parameters": {
+          "class": "high"
+        }
+      }
+    ],
+    "message": "unauthorized attempt",
+    "timestamp_epoch": 1065935655,
+    "timestamp_epoch_utc": 1065910455
+  }
+]
+```
+### Syslog strings (RFC 3164)
+```bash
+cat syslog.txt | jc --syslog-bsd -p
+```
+```json
+[
+  {
+    "priority": 34,
+    "date": "Oct 11 22:14:15",
+    "hostname": "mymachine",
+    "tag": "su",
+    "content": "'su root' failed for lonvick on /dev/pts/8"
+  }
+]
+```
 ### systemctl
 ```bash
 systemctl -a | jc --systemctl -p          # or:  jc -p systemctl -a

diff --git a/README.md b/README.md
@@ -13,9 +13,9 @@ for an example.
 # JC
 JSON Convert
 
-`jc` JSONifies the output of many CLI tools and file-types for easier parsing in
-scripts. See the [**Parsers**](#parsers) section for supported commands and
-file-types.
+`jc` JSONifies the output of many CLI tools, file-types, and common strings
+for easier parsing in scripts. See the [**Parsers**](#parsers) section for
+supported commands, file-types, and strings.
 ```bash
 dig example.com | jc --dig
 ```
@@ -93,6 +93,7 @@ Use Cases:
 - [Ansible command output parsing](https://blog.kellybrazil.com/2020/08/30/parsing-command-output-in-ansible-with-jc/)
 - [Saltstack command output parsing](https://blog.kellybrazil.com/2020/09/15/parsing-command-output-in-saltstack-with-jc/)
 - [Nornir command output parsing](https://blog.kellybrazil.com/2020/12/09/parsing-command-output-in-nornir-with-jc/)
+- [FortiSOAR command output parsing](https://docs.fortinet.com/document/fortisoar/1.0.0/jc-parse-command-output/323/jc-parse-command-output-v1-0-0)
 
 ## Installation
 There are several ways to get `jc`. You can install via `pip`, OS package
@@ -120,6 +121,7 @@ pip3 install jc
 | macOS                                | `brew install jc`                                                             |
 | FreeBSD                              | `portsnap fetch update && cd /usr/ports/textproc/py-jc && make install clean` |
 | Ansible filter plugin                | `ansible-galaxy collection install community.general`                         |
+| FortiSOAR connector                  | Install from FortiSOAR Connector Marketplace                                  |
 
 > For more OS Packages, see https://repology.org/project/jc/versions.
 
@@ -155,6 +157,8 @@ option.
 | `   --asciitable` | ASCII and Unicode table parser                          | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/asciitable)     |
 | ` --asciitable-m` | multi-line ASCII and Unicode table parser               | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/asciitable_m)   |
 | `        --blkid` | `blkid` command parser                                  | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/blkid)          |
+| `          --cef` | CEF string parser                                       | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/cef)            |
+| `        --cef-s` | CEF string streaming parser                             | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/cef_s)          |
 | `        --chage` | `chage --list` command parser                           | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/chage)          |
 | `        --cksum` | `cksum` and `sum` command parser                        | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/cksum)          |
 | `      --crontab` | `crontab` command and file parser                       | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/crontab)        |
@@ -189,10 +193,11 @@ option.
 | `          --ini` | INI file parser                                         | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/ini)            |
 | `       --iostat` | `iostat` command parser                                 | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/iostat)         |
 | `     --iostat-s` | `iostat` command streaming parser                       | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/iostat_s)       |
+| `   --ip-address` | IPv4 and IPv6 Address string parser                     | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/ip_address)     |
 | `     --iptables` | `iptables` command parser                               | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/iptables)       |
 | ` --iso-datetime` | ISO 8601 Datetime string parser                         | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/iso_datetime)   |
 | `      --iw-scan` | `iw dev [device] scan` command parser                   | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/iw_scan)        |
-| ` --jar-manifest` | MANIFEST.MF file parser                                 | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/jar_manifest)   |
+| ` --jar-manifest` | Java MANIFEST.MF file parser                            | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/jar_manifest)   |
 | `         --jobs` | `jobs` command parser                                   | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/jobs)           |
 | `          --jwt` | JWT string parser                                       | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/jwt)            |
 | `           --kv` | Key/Value file parser                                   | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/kv)             |
@@ -204,6 +209,7 @@ option.
 | `         --lsof` | `lsof` command parser                                   | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/lsof)           |
 | `        --lsusb` | `lsusb` command parser                                  | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/lsusb)          |
 | `          --m3u` | M3U and M3U8 file parser                                | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/m3u)            |
+| `        --mdadm` | `mdadm` command parser                                  | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/mdadm)          |
 | `        --mount` | `mount` command parser                                  | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/mount)          |
 | `       --mpstat` | `mpstat` command parser                                 | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/mpstat)         |
 | `     --mpstat-s` | `mpstat` command streaming parser                       | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/mpstat_s)       |
@@ -217,6 +223,7 @@ option.
 | `       --ping-s` | `ping` and `ping6` command streaming parser             | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/ping_s)         |
 | `     --pip-list` | `pip list` command parser                               | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/pip_list)       |
 | `     --pip-show` | `pip show` command parser                               | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/pip_show)       |
+| `        --plist` | PLIST file parser                                       | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/plist)          |
 | `     --postconf` | `postconf -M` command parser                            | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/postconf)       |
 | `           --ps` | `ps` command parser                                     | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/ps)             |
 | `        --route` | `route` command parser                                  | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/route)          |
@@ -229,14 +236,18 @@ option.
 | `         --stat` | `stat` command parser                                   | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/stat)           |
 | `       --stat-s` | `stat` command streaming parser                         | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/stat_s)         |
 | `       --sysctl` | `sysctl` command parser                                 | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/sysctl)         |
+| `       --syslog` | Syslog RFC 5424 string parser                           | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/syslog)         |
+| `     --syslog-s` | Syslog RFC 5424 string streaming parser                 | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/syslog_s)       |
+| `   --syslog-bsd` | Syslog RFC 3164 string parser                           | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/syslog_bsd)     |
+| ` --syslog-bsd-s` | Syslog RFC 3164 string streaming parser                 | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/syslog_bsd_s)   |
 | `    --systemctl` | `systemctl` command parser                              | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/systemctl)      |
 | ` --systemctl-lj` | `systemctl list-jobs` command parser                    | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/systemctl_lj)   |
 | ` --systemctl-ls` | `systemctl list-sockets` command parser                 | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/systemctl_ls)   |
 | `--systemctl-luf` | `systemctl list-unit-files` command parser              | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/systemctl_luf)  |
 | `   --systeminfo` | `systeminfo` command parser                             | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/systeminfo)     |
 | `         --time` | `/usr/bin/time` command parser                          | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/time)           |
 | `  --timedatectl` | `timedatectl status` command parser                     | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/timedatectl)    |
-| `    --timestamp` | UNIX Epoch Timestamp string parser                      | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/timestamp)      |
+| `    --timestamp` | Unix Epoch Timestamp string parser                      | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/timestamp)      |
 | `          --top` | `top -b` command parser                                 | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/top)            |
 | `        --top-s` | `top -b` command streaming parser                       | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/top_s)          |
 | `    --tracepath` | `tracepath` and `tracepath6` command parser             | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/tracepath)      |
@@ -269,6 +280,7 @@ option.
 | `-d`  | `--debug`       | Debug mode. Prints trace messages if parsing issues are encountered (use`-dd` for verbose debugging)                |
 | `-h`  | `--help`        | Help. Use `jc -h --parser_name` for parser documentation                                                            |
 | `-m`  | `--monochrome`  | Monochrome output                                                                                                   |
+| `-M`  | `--meta-out`    | Add metadata to output including timestamp, parser name, magic command, magic command exit code, etc.               |                                                                        |
 | `-p`  | `--pretty`      | Pretty format the JSON output                                                                                       |
 | `-q`  | `--quiet`       | Quiet mode. Suppresses parser warning messages (use `-qq` to ignore streaming parser errors)                        |
 | `-r`  | `--raw`         | Raw output. Provides more literal output, typically with string values and no additional semantic processing        |
@@ -432,15 +444,16 @@ Local plugins may override default parsers.
 
 #### Locale
 
-For best results set the `LANG` locale environment variable to `C` or
-`en_US.UTF-8`. For example, either by setting directly on the command-line:
+For best results set the locale environment variables to `C` or
+`en_US.UTF-8` by modifying the `LC_ALL` variable:
 ```
-$ LANG=C date | jc --date
+$ LC_ALL=C date | jc --date
 ```
 
-or by exporting to the environment before running commands:
+You can also set the locale variables individually:
 ```
 $ export LANG=C
+$ export LC_NUMERIC=C
 ```
 
 On some older systems UTF-8 output will be downgraded to ASCII with `\\u`