Merge pull request #88 from mountaindude/master

v4.1.0 RC1

.codeclimate.yml

@@ -1,4 +1,14 @@
 version: "2"         # required to adjust maintainability checks
+plugins:
+  eslint:
+    enabled: true
+  fixme:
+    enabled: true
+  markdownlint:
+    enabled: true
+  nodesecurity:
+    enabled: true
+
 checks:
   argument-count:
     config:
@@ -17,7 +27,7 @@ checks:
       threshold: 20
   method-lines:
     config:
-      threshold: 25
+      threshold: 50
   nested-control-flow:
     config:
       threshold: 4
@@ -31,24 +41,5 @@ checks:
     config:
       threshold: # language-specific defaults. an override will affect all languages.
 
-plugins:
-  eslint:
-    enabled: true
-    # channel: "eslint-6"
-#   csslint:
-#     enabled: true
-  markdownlint:
-    enabled: true
-
 exclude_patterns:
-#   - "config/"
-  - "db/"
-  - "dist/"
-  - "features/"
   - "**/node_modules/"
-  - "script/"
-  - "**/spec/"
-  - "**/test/"
-  - "**/tests/"
-  - "Tests/"
-  - "**/vendor/"

README.md

@@ -1,6 +1,7 @@
 # Butler
 
 [![Build Status](https://cloud.drone.io/api/badges/ptarmiganlabs/butler/status.svg)](https://cloud.drone.io/ptarmiganlabs/butler)
+[![Maintainability](https://api.codeclimate.com/v1/badges/cb33990248ffbaaf7526/maintainability)](https://codeclimate.com/github/ptarmiganlabs/butler/maintainability)
 
 ![Butler](icon.png)
 \

changelog.md

@@ -1,6 +1,23 @@
 # Change log
 
-## 4.0
+## 4.1.0
+
+### New features
+
+- File copy method in the REST API.
+- Added Qlik Sense app "Butler 4.1 demo app" which contains examples of how Butler's API can be called from Sense load scripts.
+
+### Fixes and patches
+
+- Updated dependencies to latest versions.
+- Minor stability changes/improvements to filemove and filedelete API methods.
+- Change parsing of config file to make Butler more forgiving if some parts of the config file are missing.
+
+### Breaking changes
+
+None.
+
+## 4.0.0
 
 ### New features
 

src/butler.js

@@ -129,82 +129,88 @@ restifySwaggerJsdoc.createSwaggerPage({
 //     restServer.get({ path: '/v4/getdiskspace' }, rest.getDiskSpace.respondGET_getDiskSpace);
 // }
 
-if (globals.config.get('Butler.restServerEndpointsEnable.apiListEnbledEndpoints')) {
-    globals.logger.debug('Registering REST endpoint PGETUT /v4/configfile/endpointsenabled');
+// TODO First test if config entry exists => Make Butler less picky about new formats in the config file
+if (globals.config.has('Butler.restServerEndpointsEnable.apiListEnbledEndpoints') && globals.config.get('Butler.restServerEndpointsEnable.apiListEnbledEndpoints')) {
+    globals.logger.debug('Registering REST endpoint GET /v4/configfile/endpointsenabled');
     restServer.get({ path: '/v4/configfile/endpointsenabled' }, rest.api.respondGET_configFileListEnbledEndpoints);
 }
 
-if (globals.config.get('Butler.restServerEndpointsEnable.fileDelete')) {
+if (globals.config.has('Butler.restServerEndpointsEnable.fileDelete') && globals.config.get('Butler.restServerEndpointsEnable.fileDelete')) {
     globals.logger.debug('Registering REST endpoint DELETE /v4/filedelete');
     restServer.del({ path: '/v4/filedelete' }, rest.disk_utils.respondPUT_fileDelete);
 }
 
-if (globals.config.get('Butler.restServerEndpointsEnable.fileMove')) {
+if (globals.config.has('Butler.restServerEndpointsEnable.fileMove') && globals.config.get('Butler.restServerEndpointsEnable.fileMove')) {
     globals.logger.debug('Registering REST endpoint PUT /v4/filemove');
     restServer.put({ path: '/v4/filemove' }, rest.disk_utils.respondPUT_fileMove);
 }
 
-if (globals.config.get('Butler.restServerEndpointsEnable.activeUserCount')) {
+if (globals.config.has('Butler.restServerEndpointsEnable.fileCopy') && globals.config.get('Butler.restServerEndpointsEnable.fileCopy')) {
+    globals.logger.debug('Registering REST endpoint PUT /v4/filecopy');
+    restServer.put({ path: '/v4/filecopy' }, rest.disk_utils.respondPUT_fileCopy);
+}
+
+if (globals.config.has('Butler.restServerEndpointsEnable.activeUserCount') && globals.config.get('Butler.restServerEndpointsEnable.activeUserCount')) {
     globals.logger.debug('Registering REST endpoint GET /v4/activeusercount');
     restServer.get({ path: '/v4/activeusercount' }, rest.activeUserCount.respondGET_activeUserCount);
 }
 
-if (globals.config.get('Butler.restServerEndpointsEnable.activeUsers')) {
+if (globals.config.has('Butler.restServerEndpointsEnable.activeUsers') && globals.config.get('Butler.restServerEndpointsEnable.activeUsers')) {
     globals.logger.debug('Registering REST endpoint GET /v4/activeusers');
     restServer.get({ path: '/v4/activeusers' }, rest.activeUsers.respondGET_activeUsers);
 }
 
-if (globals.config.get('Butler.restServerEndpointsEnable.slackPostMessage')) {
+if (globals.config.has('Butler.restServerEndpointsEnable.slackPostMessage') && globals.config.get('Butler.restServerEndpointsEnable.slackPostMessage')) {
     globals.logger.debug('Registering REST endpoint PUT /v4/slackpostmessage');
     restServer.put({ path: '/v4/slackpostmessage' }, rest.slackPostMessage.respondPUT_slackPostMessage);
 }
 
-if (globals.config.get('Butler.restServerEndpointsEnable.createDir')) {
+if (globals.config.has('Butler.restServerEndpointsEnable.createDir') && globals.config.get('Butler.restServerEndpointsEnable.createDir')) {
     globals.logger.debug('Registering REST endpoint POST /v4/createdir');
     restServer.post({ path: '/v4/createdir' }, rest.disk_utils.respondPOST_createDir);
 }
 
-if (globals.config.get('Butler.restServerEndpointsEnable.createDirQVD')) {
+if (globals.config.has('Butler.restServerEndpointsEnable.createDirQVD') && globals.config.get('Butler.restServerEndpointsEnable.createDirQVD')) {
     globals.logger.debug('Registering REST endpoint POST /v4/createdirqvd');
     restServer.post({ path: '/v4/createdirqvd' }, rest.disk_utils.respondPOST_createDirQVD);
 }
 
-if (globals.config.get('Butler.restServerEndpointsEnable.mqttPublishMessage')) {
+if (globals.config.has('Butler.restServerEndpointsEnable.mqttPublishMessage') && globals.config.get('Butler.restServerEndpointsEnable.mqttPublishMessage')) {
     globals.logger.debug('Registering REST endpoint PUT /v4/mqttpublishmessage');
     restServer.put({ path: '/v4/mqttpublishmessage' }, rest.mqttPublishMessage.respondPUT_mqttPublishMessage);
 }
 
-if (globals.config.get('Butler.restServerEndpointsEnable.senseStartTask')) {
+if (globals.config.has('Butler.restServerEndpointsEnable.senseStartTask') && globals.config.get('Butler.restServerEndpointsEnable.senseStartTask')) {
     globals.logger.debug('Registering REST endpoint PUT /v4/sensestarttask');
     restServer.put({ path: '/v4/reloadtask/:taskId/start' }, rest.senseStartTask.respondPUT_senseStartTask);
 }
 
-if (globals.config.get('Butler.restServerEndpointsEnable.senseAppDump')) {
+if (globals.config.has('Butler.restServerEndpointsEnable.senseAppDump') && globals.config.get('Butler.restServerEndpointsEnable.senseAppDump')) {
     globals.logger.debug('Registering REST endpoint GET /v4/senseappdump');
     restServer.get({ path: '/v4/senseappdump/:appId' }, rest.senseAppDump.respondGET_senseAppDump);
 }
 
-if (globals.config.get('Butler.restServerEndpointsEnable.senseListApps')) {
+if (globals.config.has('Butler.restServerEndpointsEnable.senseListApps') && globals.config.get('Butler.restServerEndpointsEnable.senseListApps')) {
     globals.logger.debug('Registering REST endpoint GET /v4/senselistapps');
     restServer.get({ path: '/v4/senselistapps' }, rest.senseListApps.respondGET_senseListApps);
 }
 
-if (globals.config.get('Butler.restServerEndpointsEnable.butlerping')) {
+if (globals.config.has('Butler.restServerEndpointsEnable.butlerping') && globals.config.get('Butler.restServerEndpointsEnable.butlerping')) {
     globals.logger.debug('Registering REST endpoint GET /v4/butlerping');
     restServer.get({ path: '/v4/butlerping' }, rest.butlerPing.respondGET_butlerPing);
 }
 
-if (globals.config.get('Butler.restServerEndpointsEnable.base62ToBase16')) {
+if (globals.config.has('Butler.restServerEndpointsEnable.base62ToBase16') && globals.config.get('Butler.restServerEndpointsEnable.base62ToBase16')) {
     globals.logger.debug('Registering REST endpoint GET /v4/base62tobase16');
     restServer.get({ path: '/v4/base62tobase16' }, rest.base62ToBase16.respondGET_base62ToBase16);
 }
 
-if (globals.config.get('Butler.restServerEndpointsEnable.base16ToBase62')) {
+if (globals.config.has('Butler.restServerEndpointsEnable.base16ToBase62') && globals.config.get('Butler.restServerEndpointsEnable.base16ToBase62')) {
     globals.logger.debug('Registering REST endpoint GET /v4/base16tobase62');
     restServer.get({ path: '/v4/base16tobase62' }, rest.base16ToBase62.respondGET_base16ToBase62);
 }
 
-if (globals.config.get('Butler.restServerEndpointsEnable.keyValueStore')) {
+if (globals.config.has('Butler.restServerEndpointsEnable.keyValueStore') && globals.config.get('Butler.restServerEndpointsEnable.keyValueStore')) {
     globals.logger.debug('Registering REST endpoint GET /v4/keyvaluenamespaces');
     restServer.get({ path: '/v4/keyvaluesnamespaces' }, rest.keyValueStore.respondGET_keyvaluesnamespaces);
 
@@ -224,32 +230,32 @@ if (globals.config.get('Butler.restServerEndpointsEnable.keyValueStore')) {
     restServer.del({ path: '/v4/keyvalues/:namespace' }, rest.keyValueStore.respondDELETE_keyvaluesDelete);
 }
 
-if (globals.config.get('Butler.scheduler.enable')) {
-    if (globals.config.get('Butler.restServerEndpointsEnable.scheduler.createNewSchedule')) {
+if (globals.config.has('Butler.scheduler.enable') && globals.config.get('Butler.scheduler.enable')) {
+    if (globals.config.has('Butler.restServerEndpointsEnable.scheduler.createNewSchedule') && globals.config.get('Butler.restServerEndpointsEnable.scheduler.createNewSchedule')) {
         globals.logger.debug('Registering REST endpoint POST /v4/schedules');
 
         restServer.post({ path: '/v4/schedules' }, rest.scheduler.respondPOST_schedules);
     }
 
-    if (globals.config.get('Butler.restServerEndpointsEnable.scheduler.getSchedule')) {
+    if (globals.config.has('Butler.restServerEndpointsEnable.scheduler.getSchedule') && globals.config.get('Butler.restServerEndpointsEnable.scheduler.getSchedule')) {
         globals.logger.debug('Registering REST endpoint GET /v4/schedules');
 
         restServer.get({ path: '/v4/schedules' }, rest.scheduler.respondGET_schedules);
     }
 
-    if (globals.config.get('Butler.restServerEndpointsEnable.scheduler.deleteSchedule')) {
+    if (globals.config.has('Butler.restServerEndpointsEnable.scheduler.deleteSchedule') && globals.config.get('Butler.restServerEndpointsEnable.scheduler.deleteSchedule')) {
         globals.logger.debug('Registering REST endpoint DELETE /v4/schedules');
 
         restServer.del({ path: '/v4/schedules/:scheduleId' }, rest.scheduler.respondDELETE_schedules);
     }
 
-    if (globals.config.get('Butler.restServerEndpointsEnable.scheduler.startSchedule')) {
+    if (globals.config.has('Butler.restServerEndpointsEnable.scheduler.startSchedule') && globals.config.get('Butler.restServerEndpointsEnable.scheduler.startSchedule')) {
         globals.logger.debug('Registering REST endpoint POST /v4/schedulestart');
 
         restServer.put({ path: '/v4/schedules/:scheduleId/start' }, rest.scheduler.respondPUT_schedulesStart);
     }
 
-    if (globals.config.get('Butler.restServerEndpointsEnable.scheduler.stopSchedule')) {
+    if (globals.config.has('Butler.restServerEndpointsEnable.scheduler.stopSchedule') && globals.config.get('Butler.restServerEndpointsEnable.scheduler.stopSchedule')) {
         globals.logger.debug('Registering REST endpoint POST /v4/schedulestop');
 
         restServer.put({ path: '/v4/schedules/:scheduleId/stop' }, rest.scheduler.respondPUT_schedulesStop);

src/config/production_template.yaml

@@ -81,18 +81,33 @@ Butler:
     serverHost: <FQDN or IP (or localhost) of server where Butler is running>
     serverPort: 8080
 
-  fileMoveApprovedDirectories:                        # List of directories between which file moves via the REST API can be done.
+  # List of directories between which file copying via the REST API can be done.
+  # Butler will try to clean up messy paths like this one, which resolves to /Users/goran/butler-test-dir1
+  # How? First you have /Users/goran/butler-test-dir1//abc which cleans up to /Users/goran/butler-test-dir1/abc, 
+  # then up one level (..).
+  fileCopyApprovedDirectories:                        
+    - fromDirectory: /Users/goran/butler-test-dir1//abc//..     
+      toDirectory: /Users/goran/butler-test-dir2
+    - fromDirectory: /Users/goran/butler-test-dir2
+      toDirectory: /Users/goran/butler-test-dir1
+    - fromDirectory: /from/some/directory2
+      toDirectory: /to/some/directory2
+
+  # List of directories between which file moves via the REST API can be done.
+  fileMoveApprovedDirectories:                        
     - fromDirectory: /Users/goran/butler-test-dir1//abc//..
       toDirectory: /Users/goran/butler-test-dir2
     - fromDirectory: /Users/goran/butler-test-dir2
       toDirectory: /Users/goran/butler-test-dir1
     - fromDirectory: /from/some/directory2
       toDirectory: /to/some/directory2
 
-  fileDeleteApprovedDirectories:                      # List of directories in which file deletes via the REST API can be done.
+  # List of directories in which file deletes via the REST API can be done.
+  fileDeleteApprovedDirectories:                      
     - /Users/goran/butler-test-dir1
     - /Users/goran/butler-test-dir1//abc//..
     - /Users/goran/butler-test-dir2
+
   # Enable/disable individual REST API endpoints. Set config item below to true to enable that endpoint.
   restServerEndpointsEnable:
     activeUserCount: false
@@ -105,6 +120,7 @@ Butler:
     createDirQVD: false
     fileDelete: false
     fileMove: false
+    fileCopy: false
     keyValueStore: false
     mqttPublishMessage: false
     scheduler:

src/docker-compose.yml

@@ -2,7 +2,7 @@
 version: '3.3'
 services:
   butler:
-    image: ptarmiganlabs/butler:4.0.0
+    image: ptarmiganlabs/butler:4.1.0
     container_name: butler
     restart: always
     ports:

src/globals.js

@@ -154,6 +154,25 @@ var udp_port_take_failure = config.get('Butler.udpServerConfig.portTaskFailure')
 // Folder under which QVD folders are to be created
 var qvdFolder = config.get('Butler.configDirectories.qvdPath');
 
+// Load approved fromDir and toDir for fileCopy operation
+var fileCopyDirectories = [];
+
+if (config.has('Butler.fileCopyApprovedDirectories')) {
+    config.get('Butler.fileCopyApprovedDirectories').forEach(element => {
+        logger.verbose(`fileCopy directories from config file: ${JSON.stringify(element, null, 2)}`);
+
+        let newDirCombo = {
+            fromDir: path.normalize(element.fromDirectory),
+            toDir: path.normalize(element.toDirectory),
+        };
+
+        logger.info(`Adding normalized fileCopy directories ${JSON.stringify(newDirCombo, null, 2)}`);
+
+        fileCopyDirectories.push(newDirCombo);
+    });
+}
+
+
 // Load approved fromDir and toDir for fileMove operation
 var fileMoveDirectories = [];
 
@@ -305,6 +324,7 @@ module.exports = {
     configSchedule,
     initInfluxDB,
     influx,
+    fileCopyDirectories,
     fileMoveDirectories,
     fileDeleteDirectories,
     endpointsEnabled,