option to define network to run stanchion on

basho · Mar 1, 2023 · 986d615 · 986d615
1 parent 2a15bc0
commit 986d615
Show file tree

Hide file tree

Showing 14 changed files with 85 additions and 70 deletions.
diff --git a/apps/riak_cs/priv/riak_cs.schema b/apps/riak_cs/priv/riak_cs.schema
@@ -56,14 +56,6 @@
   {validators, ["valid_host"]}
 ]}.
 
-%% @doc Configuration for access to request
-%% serialization service
-{mapping, "stanchion_host", "riak_cs.stanchion_host", [
-  {default, {"{{stanchion_host}}", {{stanchion_port}} }},
-  {datatype, [fqdn, ip]},
-  {validators, ["valid_host"]}
-]}.
-
 %% @doc Default cert location for https can be overridden
 %% with the ssl config variable, for example:
 {mapping, "ssl.certfile", "riak_cs.ssl.certfile", [
@@ -79,14 +71,19 @@
 ]}.
 
 
-
 %% @doc Stanchion http/https port and IP address to listen at
 {mapping, "stanchion.listener", "riak_cs.stanchion_listener", [
   {default, {"{{stanchion_ip}}", {{stanchion_port}} }},
   {datatype, ip},
   {validators, ["valid_host"]}
 ]}.
 
+%% @doc Netmask to use when selecting which network to place stanchion on
+{mapping, "stanchion_netmask", "riak_cs.stanchion_netmask", [
+  {default, "{{stanchion_netmask}}"},
+  {datatype, string}
+]}.
+
 %% @doc SSL configuration for access to request serialization
 %% service. With `on`, Riak CS connects to Stanchion with SSL.
 {mapping, "stanchion.ssl", "riak_cs.stanchion_ssl", [

diff --git a/apps/riak_cs/src/riak_cs_config.erl b/apps/riak_cs/src/riak_cs_config.erl
@@ -62,14 +62,15 @@
          max_key_length/0,
          read_before_last_manifest_write/0,
          region/0,
-         stanchion/0,
          set_stanchion/2, set_stanchion/3,
          use_2i_for_storage_calc/0,
          detailed_storage_calc/0,
          quota_modules/0,
          active_delete_threshold/0,
          fast_user_get/0,
          root_host/0,
+         stanchion/0,
+         stanchion_netmask/0,
          stanchion_hosting_mode/0,
          tussle_voss_riak_host/0
         ]).
@@ -377,10 +378,15 @@ max_key_length() ->
 %% @doc Return `stanchion' configuration data.
 -spec stanchion() -> {string(), pos_integer(), boolean()}.
 stanchion() ->
-    {ok, {Host, Port}} = application:get_env(riak_cs, stanchion_host),
-    {ok, SSL}  = application:get_env(riak_cs, stanchion_ssl),
+    {Host, Port} = application:get_env(riak_cs, stanchion_host, {"127.0.0.1", 8085}),
+    SSL  = application:get_env(riak_cs, stanchion_ssl, false),
     {Host, Port, SSL}.
 
+-spec stanchion_netmask() -> string().
+stanchion_netmask() ->
+    {ok, A} = application:get_env(riak_cs, stanchion_netmask),
+    A.
+
 -spec set_stanchion(string(), inet:port()) -> ok.
 set_stanchion(Host, Port) ->
     application:set_env(riak_cs, stanchion_host, {Host, Port}),

diff --git a/apps/riak_cs/src/riak_cs_utils.erl b/apps/riak_cs/src/riak_cs_utils.erl
@@ -72,7 +72,8 @@
          stanchion_data/0,
          camel_case/1,
          capitalize/1,
-         this_host_addr/0
+         this_host_addresses/0,
+         select_addr_for_stanchion/0
         ]).
 
 -include("riak_cs.hrl").
@@ -536,35 +537,58 @@ capitalize("") -> "";
 capitalize([H|T]) -> string:to_upper([H]) ++ T.
 
 
-
-this_host_addr() ->
+-spec select_addr_for_stanchion() -> string().
+select_addr_for_stanchion() ->
     {ok, Ifs} = inet:getifaddrs(),
+    Mask = riak_cs_config:stanchion_netmask(),
+    {ok, {M1, M2, M3, M4}} = inet:parse_address(Mask),
     case lists:filtermap(
-           fun({_If, PL}) ->
-                   case proplists:get_value(addr, PL) of
-                       AA when AA /= undefined,
-                               AA /= {127,0,0,1},
-                               AA /= {0,0,0,0},
-                               size(AA) == 4 ->
-                           {A1, A2, A3, A4} = AA,
-                           {true, {_If, lists:flatten(io_lib:format("~b.~b.~b.~b", [A1, A2, A3, A4]))}};
-                       _ ->
+           fun({_If, IfOpts}) ->
+                   {A1, A2, A3, A4} = Addr = extract_addr(IfOpts),
+                   if (M1 band A1) > 0 andalso
+                      (M2 band A2) > 0 andalso
+                      (M3 band A3) > 0 andalso
+                      (M4 band A4) > 0 ->
+                           {true, Addr};
+                      el/=se ->
                            false
                    end
-           end, Ifs) of
-        [{If, IP}] ->
-            ?LOG_DEBUG("this host address is ~s on iface ~s", [IP, If]),
-            IP;
-        [{If, IP}|_] ->
-            logger:warning("This host has multiple network interfaces configured:"
-                           " selecting ~p on ~s", [IP, If]),
-            IP;
+           end,
+           Ifs) of
+        [{A1, A2, A3, A4}|_] ->
+            lists:flatten(io_lib:format("~b.~b.~b.~b", [A1, A2, A3, A4]));
         [] ->
-            logger:warning("This host has no network interfaces with assigned addresses:"
-                           " falling back to 127.0.0.1", []),
+            logger:warning("No network interfaces with assigned addresses matching ~s:"
+                           " falling back to 127.0.0.1", [Mask]),
             "127.0.0.1"
     end.
 
+extract_addr(IfItem) ->
+    case proplists:get_value(addr, IfItem) of
+        AA when AA /= undefined,
+                AA /= {0,0,0,0},
+                size(AA) == 4 ->
+            AA;
+        _ ->
+            {127,0,0,1}
+    end.
+
+-spec this_host_addresses() -> [string()].
+this_host_addresses() ->
+    {ok, Ifs} = inet:getifaddrs(),
+    lists:filtermap(
+      fun({_If, PL}) ->
+              case proplists:get_value(addr, PL) of
+                  AA when AA /= undefined,
+                          AA /= {0,0,0,0},
+                          size(AA) == 4 ->
+                      {A1, A2, A3, A4} = AA,
+                      {true, lists:flatten(io_lib:format("~b.~b.~b.~b", [A1, A2, A3, A4]))};
+                  _ ->
+                      false
+              end
+      end, Ifs).
+
 -ifdef(TEST).
 
 camel_case_test() ->

diff --git a/apps/riak_cs/src/stanchion_migration.erl b/apps/riak_cs/src/stanchion_migration.erl
@@ -37,18 +37,18 @@
 -spec validate_stanchion() -> ok.
 validate_stanchion() ->
     {ConfiguredIP, ConfiguredPort, _Ssl} = riak_cs_config:stanchion(),
-    logger:info("validate_stanchion: ~p", [{ConfiguredIP, ConfiguredPort}]),
+    logger:debug("validate_stanchion: ~p", [{ConfiguredIP, ConfiguredPort}]),
     case read_stanchion_data() of
         {ok, {{Host, Port}, Node}}
           when Host == ConfiguredIP,
                Port == ConfiguredPort,
                Node == node() ->
-            logger:info("validate_stanchion: matching data read"),
+            logger:debug("validate_stanchion: matching data read"),
             ok;
         {ok, {{Host, Port}, Node}} ->
             logger:info("stanchion details updated: ~s:~p on ~s", [Host, Port, Node]),
-            case riak_cs_utils:this_host_addr() of
-                ConfiguredIP when node() == Node ->
+            case lists:member(ConfiguredIP, riak_cs_utils:this_host_addresses()) of
+                true when node() == Node ->
                     stop_stanchion_here(),
                     ok;
                 _ ->
@@ -57,21 +57,19 @@ validate_stanchion() ->
             apply_stanchion_details({Host, Port});
         {error, notfound} ->
             logger:info("no previously saved stanchion details; adopting stanchion here"),
-            apply_stanchion_details({ConfiguredIP, ConfiguredPort}),
-            start_stanchion_here(),
-            ok = save_stanchion_data({ConfiguredIP, ConfiguredPort})
+            adopt_stanchion()
     end.
 
 
 -spec adopt_stanchion() -> ok | {error, stanchion_not_relocatable}.
 adopt_stanchion() ->
     case riak_cs_config:stanchion_hosting_mode() of
         auto ->
-            ThisHostAddr = riak_cs_utils:this_host_addr(),
+            Addr = riak_cs_utils:select_addr_for_stanchion(),
             {ok, {_IP, Port}} = application:get_env(riak_cs, stanchion_listener),
             start_stanchion_here(),
-            ok = save_stanchion_data({ThisHostAddr, Port}),
-            apply_stanchion_details({ThisHostAddr, Port}),
+            ok = save_stanchion_data({Addr, Port}),
+            apply_stanchion_details({Addr, Port}),
             ok;
         M ->
             logger:error("Riak CS stanchion_hosting_mode is ~s. Cannot adopt stanchion.", [M]),

diff --git a/apps/riak_cs/src/stanchion_sup.erl b/apps/riak_cs/src/stanchion_sup.erl
@@ -49,18 +49,6 @@ ensure_service_bucket_props(Pbc) ->
 -spec init([]) -> {ok, {supervisor:sup_flags(), [supervisor:child_spec()]}}.
 init([]) ->
     Children = [],
-    %% stanchion webmachine to be added to this sup on demand
-    %% {ok, Mode} = application:get_env(riak_cs, stanchion_hosting_mode),
-    %% ThisHostAddr = riak_cs_utils:this_host_addr(),
-    %% case stanchion_migration:do_we_get_to_run_stanchion(Mode, ThisHostAddr) of
-    %%     {use_saved, HostPort} ->
-    %%         ok = stanchion_migration:apply_stanchion_details(HostPort),
-    %%         [];
-    %%     use_ours ->
-    %%         {ok, {_IP, Port}} = application:get_env(riak_cs, stanchion_listener),
-    %%         ok = stanchion_migration:save_stanchion_data({ThisHostAddr, Port}),
-    %%         stanchion_process_specs()
-    %% end,
     {ok, {#{strategy => one_for_one,
             intensity => 10,
             period => 10}, Children

diff --git a/config/sys.config.defaults b/config/sys.config.defaults
@@ -7,12 +7,13 @@ RIAK_HOST="127.0.0.1"
 RIAK_PORT=8087
 RCS_LISTENER_IP="0.0.0.0"
 RCS_LISTENER_PORT=8080
+STANCHION_HOSTING_MODE=auto
 STANCHION_LISTENER_IP="0.0.0.0"
 STANCHION_LISTENER_PORT=8085
-
-STANCHION_HOST="127.0.0.1"
 STANCHION_PORT=8085
 STANCHION_SSL=false
+STANCHION_NETMASK="255.255.255.255"
+TUSSLE_VOSS_RIAK_HOST=auto
 AUTH_V4_ENABLED=true
 AUTH_BYPASS=false
 

diff --git a/config/sys.docker.config.src b/config/sys.docker.config.src
@@ -32,13 +32,15 @@
       {auth_bypass, ${AUTH_BYPASS}},
       {admin_key, ${ADMIN_KEY}},
       {anonymous_user_creation, ${ANONYMOUS_USER_CREATION}},
-      {stanchion_host,{${STANCHION_HOST}, ${STANCHION_PORT}}},
       {riak_host, {${RIAK_HOST}, ${RIAK_PORT}}},
       {listener, {${RCS_LISTENER_IP}, ${RCS_LISTENER_PORT}}},
+      {stanchion_hosting_mode, ${STANCHION_HOSTING_MODE}},
       {stanchion_listener, ${STANCHION_LISTENER_IP}, ${STANCHION_LISTENER_PORT}},
       {stanchion_ssl, ${STANCHION_SSL}},
       {stanchion_ssl_certfile, ${STANCHION_SSL_CERTFILE}},
       {stanchion_ssl_keyfile, ${STANCHION_SSL_KEYFILE}},
+      {stanchion_netmask, ${STANCHION_NETMASK}},
+      {tussle_voss_riak_host, ${TUSSLE_VOSS_RIAK_HOST}}
       {connection_pools,[{request_pool,{128,0}},{bucket_list_pool,{5,0}}]}]},
  {sasl,[{sasl_error_logger,false}]},
  {webmachine,

diff --git a/rebar.docker.config b/rebar.docker.config
@@ -19,17 +19,17 @@
        , {mochiweb, "3.1.1"}
        , {exometer_core, "v1.5.7"}
        , {uuid, "2.0.5", {pkg, uuid_erl}}
+       , {poolboy, "1.5.2"}
        , {webmachine, {git, "https://github.com/TI-Tokyo/webmachine.git", {tag, "1.11.2rc2"}}}
        , {riakc, {git, "https://github.com/basho/riak-erlang-client", {tag, "3.0.8+p1"}}}
-       , {poolboy, {git, "https://github.com/basho/poolboy", {tag, "riak_kv-3.0.0"}}}
        , {cluster_info, {git, "https://github.com/basho/cluster_info", {tag, "2.1.0"}}}
-       , {riak_repl_pb_api, {git,"https://github.com/TI-Tokyo/riak_repl_pb_api.git", {tag, "3.0.9"}}}
-       , {riak_cs_multibag, {git,"https://github.com/TI-Tokyo/riak_cs_multibag.git", {branch, "develop"}}}
+       , {riak_repl_pb_api, {git, "https://github.com/TI-Tokyo/riak_repl_pb_api.git", {tag, "3.1.0rc2"}}}
+       , {riak_cs_multibag, {git, "https://github.com/TI-Tokyo/riak_cs_multibag.git", {tag, "3.1.0rc1"}}}
        ]
 }.
 
 
-{relx, [ {release, {'riak-cs', "3.0"},
+{relx, [ {release, {'riak-cs', "3.1"},
           [ sasl
           , riak_cs
           ]
@@ -64,5 +64,3 @@
  [ {del, meck, [{erl_opts, [warnings_as_errors]}]}
  ]
 }.
-
-{dialyzer, [{plt_apps, all_deps}]}.
diff --git a/rel/pkg/alpine/vars.config b/rel/pkg/alpine/vars.config
@@ -30,6 +30,7 @@
 {auth_module,       riak_cs_s3_auth}.
 {stanchion_hosting_mode, auto}.
 {tussle_voss_riak_host, auto}.
+{stanchion_netmask, "255.255.255.255"}.
 
 {node,         "riak-cs@127.0.0.1"}.
 {crash_dump,   "{{platform_log_dir}}/erl_crash.dump"}.

diff --git a/rel/pkg/deb/vars.config b/rel/pkg/deb/vars.config
@@ -24,14 +24,14 @@
 {riak_pb_port,      8087}.
 {auth_bypass,       false}.
 {admin_key,         "admin-key"}.
-{stanchion_host,    "127.0.0.1"}.
 {stanchion_port,    8085}.
 {stanchion_ssl,     off}.
 {cs_version,        030100}.
 {rewrite_module,    riak_cs_s3_rewrite}.
 {auth_module,       riak_cs_s3_auth}.
 {stanchion_hosting_mode, auto}.
 {tussle_voss_riak_host, auto}.
+{stanchion_netmask, "255.255.255.255"}.
 
 {node,         "riak-cs@127.0.0.1"}.
 {crash_dump,   "{{platform_log_dir}}/erl_crash.dump"}.

diff --git a/rel/pkg/fbsdng/vars.config b/rel/pkg/fbsdng/vars.config
@@ -21,14 +21,14 @@
 {riak_pb_port,      8087}.
 {auth_bypass,       false}.
 {admin_key,         "admin-key"}.
-{stanchion_host,    "127.0.0.1"}.
 {stanchion_port,    8085}.
 {stanchion_ssl,     off}.
 {cs_version,        030100}.
 {rewrite_module,    riak_cs_s3_rewrite}.
 {auth_module,       riak_cs_s3_auth}.
 {stanchion_hosting_mode, auto}.
 {tussle_voss_riak_host, auto}.
+{stanchion_netmask, "255.255.255.255"}.
 
 {node,         "riak-cs@127.0.0.1"}.
 {crash_dump,   "{{platform_log_dir}}/erl_crash.dump"}.

diff --git a/rel/pkg/rpm/vars.config b/rel/pkg/rpm/vars.config
@@ -23,14 +23,14 @@
 {riak_pb_port,      8087}.
 {auth_bypass,       false}.
 {admin_key,         "admin-key"}.
-{stanchion_host,    "127.0.0.1"}.
 {stanchion_port,    8085}.
 {stanchion_ssl,     off}.
 {cs_version,        030100}.
 {rewrite_module,    riak_cs_s3_rewrite}.
 {auth_module,       riak_cs_s3_auth}.
 {stanchion_hosting_mode, auto}.
 {tussle_voss_riak_host, auto}.
+{stanchion_netmask, "255.255.255.255"}.
 
 {node,         "riak-cs@127.0.0.1"}.
 {crash_dump,   "{{platform_log_dir}}/erl_crash.dump"}.

diff --git a/rel/vars.config b/rel/vars.config
@@ -20,7 +20,6 @@
 {riak_pb_port,      8087}.
 {auth_bypass,       false}.
 {admin_key,         "admin-key"}.
-{stanchion_host,    "127.0.0.1"}.  %% access stanchion when it runs on some other node
 {stanchion_ip,      "0.0.0.0"}.
 {stanchion_port,    8085}.
 {stanchion_ssl,     off}.
@@ -29,6 +28,7 @@
 {auth_module,       riak_cs_s3_auth}.
 {stanchion_hosting_mode, auto}.
 {tussle_voss_riak_host, auto}.
+{stanchion_netmask, "255.255.255.255"}.
 
 {node,         "riak-cs@127.0.0.1"}.
 {crash_dump,   "{{platform_log_dir}}/erl_crash.dump"}.

diff --git a/rel/vars/dev_vars.config.src b/rel/vars/dev_vars.config.src
@@ -20,7 +20,6 @@
 {riak_pb_port,      @RIAKPBPORT@}.
 {auth_bypass,       false}.
 {admin_key,         "admin-key"}.
-{stanchion_host,    "127.0.0.1"}.  %% access stanchion when it runs on some other node
 {stanchion_ip,      "0.0.0.0"}.
 {stanchion_port,    8085}.
 {stanchion_ssl,     off}.
@@ -29,6 +28,7 @@
 {auth_module,       riak_cs_s3_auth}.
 {stanchion_hosting_mode, auto}.
 {tussle_voss_riak_host, auto}.
+{stanchion_netmask, "192.168.255.255"}.
 
 {node,         "@NODE@"}.
 {crash_dump,   "{{platform_log_dir}}/erl_crash.dump"}.