Copy jRuby users/groups script to new projects one (#1259)

* Copy jRuby users/groups script to new projects one

* Cleanup on dirinfo_hash assignments - feedback

Author: vt0r

cookbooks/bcpc-hadoop/attributes/hdfs.rb

@@ -123,3 +123,23 @@
   hdfs['user']['space_quota'] = '30G'
   hdfs['groups']['space_quota'] = '50G'
 end
+
+# Default HDFS Projects Configuration
+default['bcpc']['hadoop']['hdfs']['projects'].tap do |projects|
+  projects['owner'] = 'hdfs'
+  projects['group'] = 'supergroup'
+  projects['perms'] = '1771'
+  projects['space_quota'] =
+    node['bcpc']['hadoop']['hdfs']['groups']['space_quota']
+  projects['ns_quota'] = 'NO_QUOTA'
+  projects['dirinfo'] = {
+    ### EXAMPLE:
+    # project: {
+    #   owner: 'user',
+    #   group: 'group',
+    #   perms: '1710',
+    #   space_quota: '300T',
+    #   ns_quota: '10000000'
+    # }
+  }
+end

cookbooks/bcpc-hadoop/files/default/create_projects_dirs.rb

@@ -0,0 +1,88 @@
+# create_projects_dirs.rb
+# -------
+# Creates HDFS directories and manages quota (if applicable)
+# Run this in an HBase jruby shell
+# -------
+# General usage:
+# hbase shell create_projects_dirs.rb dirinfo_file
+
+include Java
+
+require 'yaml'
+
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.FileSystem
+import org.apache.hadoop.fs.Path
+import org.apache.hadoop.fs.permission.FsPermission
+import org.apache.hadoop.hdfs.protocol.HdfsConstants
+
+usage = 'hbase shell '\
+  'create_projects_dirs.rb dirinfo_file'
+banner = "usage: #{usage}\n  -- need "
+
+config = {
+  'dirinfo_file' => (ARGV[0] or raise "#{banner} dirinfo_file")
+}
+
+# hdfs file system handle
+fs = FileSystem.newInstance(Configuration.new)
+
+# load dirinfo from file
+dirinfo = YAML.load_file(config['dirinfo_file'])
+
+dirinfo.each_pair do |dir, info|
+  path = Path.new("/projects/#{dir}")
+  puts "updating path: #{path}"
+
+  # enforce ownership and permissions
+  owner = info[:owner]
+  group = info[:groups]
+  perms = FsPermission.new(info[:perms].to_i(8))
+
+  # create the directory
+  fs.mkdirs(path)
+  fs.setOwner(path, owner, group)
+  fs.setPermission(path, perms)
+
+  # Takes a string with a unit-prefix
+  # returns the result as a "long"
+  # Examples are:
+  #   string2long(600) => 600
+  #   string2long('1') => 1
+  #   string2long('1k') => 1024
+  #   string2long('30T') => 32985348833280
+  def string2long(s)
+    s = s.to_s.downcase
+    prefixes = {
+      ''  => 1,
+      'k' => 1024,
+      'm' => 1024**2,
+      'g' => 1024**3,
+      't' => 1024**4,
+      'p' => 1024**5
+    }
+
+    base = s[/[0-9.]*/].to_i
+    prefix = prefixes[s[/[kmgtp]/] || '']
+
+    base * prefix
+  end
+
+  # set space quota
+  max_quota = HdfsConstants::QUOTA_RESET
+  space_quota =
+    if info[:space_quota] == 'NO_QUOTA'
+      max_quota
+    else
+      string2long(info[:space_quota])
+    end
+  ns_quota =
+    if info[:ns_quota] == 'NO_QUOTA'
+      max_quota
+    else
+      string2long(info[:ns_quota])
+    end
+  fs.setQuota(path, ns_quota, space_quota)
+end
+
+exit

cookbooks/bcpc-hadoop/libraries/projects_dir_creation.rb

@@ -0,0 +1,53 @@
+module Bcpc_Hadoop
+  module Helper
+    require 'yaml'
+    include Chef::Mixin::ShellOut
+
+    def projects_dir_creation(dirinfo)
+      file_cache = File.join(
+        Chef::Config[:file_cache_path],
+        'cookbooks/bcpc-hadoop',
+        'files/default'
+      )
+
+      # path to creation script and caches
+      jruby_script = File.join(file_cache, 'create_projects_dirs.rb')
+      dirinfo_cache = File.join(file_cache, 'dirinfo.yml')
+
+      # Build dirinfo cache
+      # -------------------
+      # use default space/ns quota if dir-specific quota unset
+      dirinfo_hash = dirinfo.map do |dir, info|
+        [
+          dir,
+          {
+            owner: info['owner'] ||
+              node['bcpc']['hadoop']['hdfs']['projects']['owner'],
+            group: info['group'] ||
+              node['bcpc']['hadoop']['hdfs']['projects']['group'],
+            perms: info['perms'] ||
+              node['bcpc']['hadoop']['hdfs']['projects']['perms'],
+            space_quota: info['space_quota'] ||
+              node['bcpc']['hadoop']['hdfs']['projects']['space_quota'],
+            ns_quota: info['ns_quota'] ||
+              node['bcpc']['hadoop']['hdfs']['projects']['ns_quota']
+          }
+        ]
+      end.to_h
+
+      # Save dirinfo cache
+      File.open(dirinfo_cache, 'w+') { |f| f.puts(dirinfo_hash.to_yaml) }
+
+      # allow the hdfs user to read them
+      File.chmod(0o0644, jruby_script)
+      File.chmod(0o0644, dirinfo_cache)
+
+      # execute the jruby script
+      jruby_shell_cmd = "hbase shell #{jruby_script} #{dirinfo_cache}"
+      jruby_shell =
+        Mixlib::ShellOut.new(jruby_shell_cmd, user: 'hdfs', timeout: 120)
+      jruby_shell.run_command
+      jruby_shell.error!
+    end
+  end
+end

cookbooks/bcpc-hadoop/recipes/hdfs_project_directories.rb

@@ -0,0 +1,7 @@
+Chef::Resource::RubyBlock.send(:include, Bcpc_Hadoop::Helper)
+
+ruby_block 'hdfs_projects_directories' do
+  block do
+    projects_dir_creation(node['bcpc']['hadoop']['hdfs']['projects']['dirinfo'])
+  end
+end