Skip to content

Commit

Permalink
Merge pull request #422 from kmgowda/kmg-drivers-1
Browse files Browse the repository at this point in the history 
Restructure storage drivers

Signed-off-by: Keshava Munegowda <keshava.gowda@gmail.com>
  • Loading branch information
@kmgowda
kmgowda authored Sep 26, 2023
2 parents 3dcc1a7 + 9e2b928 commit b366b1f
Show file tree
Hide file tree
Showing 300 changed files with 1,496 additions and 203 deletions.
104 changes: 46 additions & 58 deletions build-drivers.gradle
View file
Original file line number Diff line number Diff line change
@@ -1,62 +1,50 @@
/**
* Copyright (c) KMG. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
*/

/* Include your driver below */
dependencies {
api project(":driver-concurrentq")
api project(":driver-file")
api project(":driver-filestream")
api project(":driver-asyncfile")
api project(":driver-hdfs")
api project(":driver-bookkeeper")
api project(":driver-rabbitmq")
api project(":driver-rocketmq")
api project(":driver-pulsar")
api project(":driver-kafka")
api project(":driver-pravega")
api project(":driver-nats")
api project(':driver-natsStream')
api project(':driver-artemis')
api project(':driver-nsq')
api project(':driver-jdbc')
api project(':driver-minio')
api project(':driver-foundationdb')
api project(':driver-fdbrecord')
api project(':driver-mongodb')
api project(':driver-rocksdb')
//api project(':driver-ignite')
api project(':driver-couchdb')
api project(':driver-hive')
api project(':driver-null')
api project(':driver-sqlite')
api project(':driver-mysql')
api project(':driver-mariadb')
api project(':driver-postgresql')
api project(':driver-derby')
api project(':driver-mssql')
api project(':driver-redpanda')
api project(':driver-csv')
api project(':driver-redis')
api project(':driver-seaweeds3')
api project(':driver-cassandra')
api project(':driver-db2')
api project(':driver-cephs3')
api project(':driver-openio')
api project(':driver-leveldb')
api project(':driver-activemq')
api project(':driver-h2')
api project(':driver-couchbase')
api project(':driver-memcached')
api project(':driver-dynamodb')
api project(':driver-exasol')
/* api project(':driver-sbktemplate') */
api project(":drivers:file")
api project(":drivers:filestream")
api project(":drivers:asyncfile")
api project(":drivers:hdfs")
api project(":drivers:bookkeeper")
api project(":drivers:rabbitmq")
api project(":drivers:rocketmq")
api project(":drivers:pulsar")
api project(":drivers:kafka")
api project(":drivers:pravega")
api project(":drivers:nats")
api project(':drivers:natsStream')
api project(':drivers:artemis')
api project(':drivers:nsq')
api project(':drivers:jdbc')
api project(':drivers:minio')
api project(':drivers:foundationdb')
api project(':drivers:fdbrecord')
api project(':drivers:mongodb')
api project(':drivers:rocksdb')
//api project(':drivers:ignite')
api project(':drivers:couchdb')
api project(':drivers:hive')
api project(':drivers:null')
api project(':drivers:sqlite')
api project(':drivers:mysql')
api project(':drivers:mariadb')
api project(':drivers:postgresql')
api project(':drivers:derby')
api project(':drivers:mssql')
api project(':drivers:redpanda')
api project(':drivers:csv')
api project(':drivers:redis')
api project(':drivers:seaweeds3')
api project(':drivers:cassandra')
api project(':drivers:db2')
api project(':drivers:cephs3')
api project(':drivers:openio')
api project(':drivers:leveldb')
api project(':drivers:activemq')
api project(':drivers:h2')
api project(':drivers:couchbase')
api project(':drivers:memcached')
api project(':drivers:dynamodb')
api project(':drivers:exasol')
/* api project(':drivers:sbktemplate') */
/* above line is a signature */
}
132 changes: 73 additions & 59 deletions build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,11 @@ gradle.rootProject {
}

allprojects {

if (project.name == "drivers") {
return
}

repositories {
mavenLocal()
mavenCentral()
Expand Down Expand Up @@ -69,9 +74,10 @@ allprojects {

def sbkProjectName = project.name
def sbkClass = project.name
if (sbkProjectName.startsWith("driver-")) {
sbkProjectName = sbkProjectName.replace( "driver-", rootProject.name + "-")
sbkClass = sbkClass.replace("driver-", "")
if (project.parent) {
if (project.parent.name == "drivers") {
sbkProjectName = rootProject.name + "-" + sbkProjectName
}
}

base {
Expand All @@ -81,9 +87,10 @@ allprojects {
version = sbkVersion

configurations.all {
exclude group: sbkGroup, module: "driver-sbktemplate"
exclude group: sbkGroup, module: "sbktemplate"
}


jar {
manifest {
attributes 'Name': sbkProjectName.toUpperCase(),
Expand Down Expand Up @@ -198,7 +205,7 @@ allprojects {
}

task generateDockerfiles() {
def driversToSkip = ["sbk-api", "perl", "sbm", "sbk-gem"]
def driversToSkip = ["sbk-api", "perl", "sbm", "sbk-gem", "drivers", "sbk-yal", "sbk-gem-yal"]
def buffer = ""
def outDir = 'build/dockers'
def outFile = sbkProjectName + "-" + sbkVersion
Expand All @@ -214,19 +221,19 @@ allprojects {
}


if (project.path.startsWith(":driver-")) {
if (project.path.startsWith(":drivers")) {
doLast {
println "generatering file : " + outDir + "/" + outFile

configurations.compileClasspath.resolvedConfiguration.resolvedArtifacts.each { artifact ->
def id = artifact.moduleVersion.id
if (id.group.toString() == sbkGroup) {
if (!driversToSkip.contains(id.name)) {
buffer += "COPY --chown=root:root " + id.name + " \${APP_HOME}/" + id.name + "\n"
buffer += "COPY --chown=root:root " + id.name + " \${APP_HOME}/drivers/" + id.name + "\n"
}
}
}
buffer += "COPY --chown=root:root " + project.name + " \${APP_HOME}/" + project.name + "\n"
buffer += "COPY --chown=root:root " + project.name + " \${APP_HOME}/drivers/" + project.name + "\n"
copy {
from "$rootDir/driver-docker-template"
into outDir
Expand All @@ -244,7 +251,7 @@ allprojects {
def id = artifact.moduleVersion.id
if (id.group.toString() == sbkGroup) {
if (!driversToSkip.contains(id.name)) {
buffer += "COPY --chown=root:root " + id.name + " \${APP_HOME}/" + id.name + "\n"
buffer += "COPY --chown=root:root " + id.name + " \${APP_HOME}/drivers/" + id.name + "\n"
}
}
}
Expand Down Expand Up @@ -277,24 +284,58 @@ allprojects {
}

task printDependencies {

doLast {
project.subprojects.findAll().each {
println "subproject : " + it.name
it.configurations.compileClasspath.resolvedConfiguration.resolvedArtifacts.each { artifact ->
def id = artifact.moduleVersion.id
if (id.group.toString() == sbkGroup) {
println "\tgroup: ${id.group}, version: ${id.version}, name: ${id.name}"
it.configurations.each {k ->
k.dependencies.each { dep ->
if (dep.group.toString() == sbkGroup) {
println "\tgroup: ${dep.group}, version: ${dep.version}, name: ${dep.name}"
def projName = dep.name
while (projName) {
def proj = project.findProject(projName)
if (!proj) {
//search inside drivers
proj = project.findProject("drivers:"+projName)
}
projName = null
if (proj) {
proj.configurations.each {x->
x.dependencies.each { y ->
if(y.group.toString() == sbkGroup) {
println "\tgroup: ${y.group}, version: ${y.version}, name: ${y.name}"
projName = y.name
}
}
}
} else {
break
}
}
}
}
}
println()
}

println("Root project : " + rootProject.name)
rootProject.configurations.each {k ->
k.dependencies.each {dep ->
if (dep.group.toString() == sbkGroup) {
println "\tgroup: ${dep.group}, version: ${dep.version}, name: ${dep.name}"
}
}
}

/*
configurations.compileClasspath.resolvedConfiguration.resolvedArtifacts.each { artifact ->
def id = artifact.moduleVersion.id
if (id.group.toString() == sbkGroup) {
println "\tgroup: ${id.group}, version: ${id.version}, name: ${id.name}"
}
}
*/
println()
}
}
Expand All @@ -305,21 +346,20 @@ task printDependencies {
another command : ./gradlew printdrivers -Preadme -Pdec (add the project description)
*/
task printDrivers {
String readmePrefix = ""
String readmePrefix = "drivers/"

doLast {
int i = 1;
println();
project.subprojects.findAll().each {
if (it.name.startsWith("driver-") && !it.name.equals("driver-template")) {
if (it.parent.name == "drivers") {
String out = it.name
String dec = it.getDescription()
if (project.hasProperty("readme")) {
String name = it.name.replace("driver-", "")
if (dec != null && project.hasProperty("dec")) {
out = "[" + name.capitalize() + "](" + readmePrefix + it.name + " \"" + dec + "\"" + ")"
out = "[" + it.name.capitalize() + "](" + readmePrefix + it.name + " \"" + dec + "\"" + ")"
} else {
out = "[" + name.capitalize() + "](" + readmePrefix + it.name + ")"
out = "[" + it.name.capitalize() + "](" + readmePrefix + it.name + ")"
}
}
printf("%4d. %s\n", i, out)
Expand Down Expand Up @@ -488,20 +528,11 @@ application {

task deleteDriver() {
doLast {
String templateName = "driver-sbktemplate"
String templateName = "sbktemplate"
String driverName = templateName

if (project.hasProperty("driver")) {
String name1 = driver
if (name1.equalsIgnoreCase("driver") || name1.equalsIgnoreCase("driver-")) {
println "driver name is missing"
return
} else if (name1.startsWith("driver-")) {
driverName = name1
} else {
driverName = "driver-" + name1
}
driverName = driverName.toLowerCase()
driverName = driver.toLowerCase()
} else {
println "driver name is missing user -Pdriver "
return
Expand All @@ -511,7 +542,7 @@ task deleteDriver() {
println "Invalid driver name"
return
}
delete "$rootDir/$driverName"
delete "$rootDir/drivers/$driverName"

def inFile1 = new File("$rootDir/settings-drivers.gradle").newInputStream()
String outText1 = ""
Expand Down Expand Up @@ -542,44 +573,28 @@ task deleteDriver() {

task addDriver(dependsOn: deleteDriver) {
doLast {
String templateName = "driver-sbktemplate"
String templateName = "sbktemplate"
String driverName = templateName
String sbkDriverName = ""
String sbkDriverName = driverName.capitalize()

if (project.hasProperty("driver")) {
String name1 = driver
if (name1.equalsIgnoreCase("driver") || name1.equalsIgnoreCase("driver-")) {
println "driver name is missing"
return
} else if (name1.startsWith("driver-")) {
driverName = name1
String[] lt = name1.split("driver-")
if (lt.length < 2) {
println "Missing driver name"
return
}
sbkDriverName = lt[1]
} else {
driverName = "driver-" + name1
sbkDriverName = name1
}
driverName = driverName.toLowerCase()
sbkDriverName = sbkDriverName.capitalize()
driverName = driver.toLowerCase()
} else {
println "driver name is missing user -Pdriver "
return
}

println "Add Driver: $driverName"
sbkDriverName = driverName.capitalize()
println "Add Driver: $sbkDriverName"
if (driverName.contains(templateName)) {
println "Invalid driver name"
return
}

copy {
from "$rootDir/$templateName"
into "$rootDir/$driverName"
from "$rootDir/drivers/$templateName"
into "$rootDir/drivers/$driverName"
from("$driverName/src/main/java/io/sbk/SbkTemplate") {
into "src/main/java/io/sbk/$sbkDriverName"
into "src/main/java/io/sbk/$driverName"
}
exclude 'build'
rename "SbkTemplate", sbkDriverName
Expand All @@ -592,7 +607,7 @@ task addDriver(dependsOn: deleteDriver) {
inFile1.eachLine {
String line = it
if (line.contains(templateName)) {
outText1 += " api project(':$driverName')\n"
outText1 += " api project(':drivers:$driverName')\n"
}
outText1 += line + "\n"
}
Expand All @@ -605,7 +620,7 @@ task addDriver(dependsOn: deleteDriver) {
inFile2.eachLine {
String line = it
if (line.contains(templateName)) {
outText2 += "include '$driverName'\n"
outText2 += "include 'drivers:$driverName'\n"
}
outText2 += line + "\n"
}
Expand Down Expand Up @@ -665,5 +680,4 @@ dependencies {
api project(":sbk-gem-yal")
}

apply from: "$rootDir/build-drivers.gradle"

apply from: "build-drivers.gradle"
2 changes: 2 additions & 0 deletions docker-template
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@ COPY --chown=root:root sbm ${APP_HOME}/sbm
COPY --chown=root:root sbk-gem ${APP_HOME}/sbk-gem
COPY --chown=root:root sbk-gem-yal ${APP_HOME}/sbk-gem-yal

RUN mkdir -p ${APP_HOME}/drivers

# Copy the SBK storage drivers
COPY@SBK

Expand Down
Loading

0 comments on commit b366b1f

Please sign in to comment.