Fred's Blog

require 'rubygems'
require 'aws/s3'
require 'fastthread'
require 'pathname'
require 'ftools'


#########################
##   SCRIPT SETTINGS   ##
#########################


# Unattended mode will not ask any questions
@unattended_mode = false


@access_key_id = ENV['AMAZON_ACCESS_KEY_ID']
@secret_access_key = ENV['AMAZON_SECRET_ACCESS_KEY']
@account_name = "myapp"

@time = Time.now
@bucket_name = "#{@account_name}_mysql_backup_#{@time.year}-#{@time.month}"
@bucket_name = "#{@account_name}_mysql_backup_#{@time.year}-#{@time.month}"

@data_dir = "/tmp"
@filename = "#{@time.year}.#{@time.month}.#{@time.day}__#{@time.hour}.#{@time.min}.#{@time.sec}"

# Array of databases to backup
# Example: ["rails_test", "rails_dev", "myapp_dev", "myapp_prod"]
@databases = []

# If this is true, backup all databases and ignore @databases array
# true / false
@all_databases = true

# Username and Password to access MYSQL
# it's good to create a user with READ only access to all databases.
# for example: GRANT SELECT ON *.* TO 'fred'@'localhost' IDENTIFIED by 'fred'
@db_username = "root"
@db_password = ""


@lines = "\n----------------------------------------------------------"

if @unattended_mode == false
  puts "Welcome!"
  puts "--------"
  puts "Program Variables:"
  puts "------------------" 
  puts "- S3 Bucket:          #{@bucket_name}"
  puts "- Local Dump Dir:     #{@data_dir}"
  puts "- Local Time of Dump: #{@time}"
  puts "- Databases:          #{@databases.join(',')}"
  puts "- All Databases?      #{@all_databases}"
  puts "- DB Username:        #{@db_username}"
  puts "- DB Password:        Not Shown"
  puts "- Unattended mode:    #{@unattended_mode}"
  puts @lines
  puts "Is this Information correct?"
end

# check if
def check_tmp_directory
  begin
    FileUtils.touch(@data_dir)
  rescue
    puts "Cannot write to temp directory: #{@data_dir}"
    exit
  end
end
  
def check_answer
  if @unattended_mode == false
    puts "Press Y to Continue or N no Cancel."
    yes_no = gets
    yes_no.chomp!
    case yes_no
      when "Y","y","Yes","yes"
        puts "Continuing"
      when "N", "n", "No", "no"
        puts "You chose to CANCEL, bye bye."
        exit
      when 'q', 'quit'
        puts "You chose to QUIT, bye bye."
        exit
      else
        puts "Invalid Answer."
        check_answer
    end
  else
    return true
  end
end

def check_settings
  if @access_key_id.to_s.empty?
    puts "FATAL: AMAZON_ACCESS_KEY_ID not set, quiting now."
    exit
  end
  if @secret_access_key.to_s.empty?
    puts "FATAL: AMAZON_SECRET_ACCESS_KEY not set, quiting now."
    exit
  end  
end

# Function to make the Database Dumps
# if db_name is nil, then backup all-databases :)
def mysqldump(file_name, db_name)
  if db_name
    Thread.new do
      if @db_password.to_s.empty?
        IO.popen(" mysqldump -u #{@db_username} #{db_name} > #{file_name}")
      else
        IO.popen(" mysqldump -u #{@db_username} -p#{@db_password} #{db_name} > #{file_name}")
      end
    end
  else
    Thread.new do
      if @db_password.to_s.empty?
        IO.popen(" mysqldump -u #{@db_username} --all_databases > #{file_name}")
      else
        IO.popen(" mysqldump -u #{@db_username} -p#{@db_password} --all_databases > #{file_name}")
      end
    end
  end
end

# Function to run the actual mysqldump command
def make_mysql_backup
  if @all_databases
    file_name = "#{@data_dir}/#{@filename}__ALL.sql"
    mysqldump(file_name,nil)
  else
    @databases.each do |db_name|
      file_name = "#{@data_dir}/#{@filename}__#{db_name}.sql"
      puts "Dumping #{db_name} into #{file_name}\n"
      mysqldump(file_name,db_name)
    end
  end
  # wait for mysqldump to finish, 
  # TODO, make some function to check this dinamically
  sleep 45
end

# Function to stablish connection
def stablish_connection
  begin
    AWS::S3::Base.establish_connection!(
      :access_key_id     => @access_key_id,
      :secret_access_key => @secret_access_key
    )
  rescue => exception
    puts @lines
    puts "There was an error: "
    puts exception.to_s
    exit
  end
  puts "Good, Connection Stablished."
end


# Function to find or create a bucket
def find_or_create_bucket
  begin
    AWS::S3::Bucket.find(@bucket_name)
  rescue
    puts "Bucket #{@bucket_name} not found."
    puts 'Creating the bucket now.'
    AWS::S3::Bucket.create(@bucket_name)
    retry
  end
  puts "Good, bucket #{@bucket_name} found."
end


# Function to send data to bucket
def send_data
  puts "Full Pathname localy is #{@data_dir}."
  @files_count = 0 
  @data_transferred = 0
  
  p = Pathname.new(@data_dir)
  p.children.each do |item|
    file_name = item.relative_path_from(Pathname.new(@data_dir)).to_s
    @files_count += 1
    @data_transferred += item.size
    puts "Putting Local File: '#{item}'"
    puts "To bucket: '#{file_name}'"
    AWS::S3::S3Object.store(file_name, open(item), @bucket_name)
  end

  puts @lines
  puts "Files Copied: #{@files_count}"
  puts "Data Transfered: #{to_file_size(@data_transferred)}"
  puts "done!"
end

# for nice printing of data transfered :)
def to_file_size(num)
  case num
  when 0 
    return "0 byte"
  when 1..1024
    return "1K"
  when 1025..1048576
    kb = num/1024.0
    return "#{f_to_dec(kb)} Kb"
  when 1024577..1049165824
    kb = num/1024.0
    mb = kb / 1024.0
    return "#{f_to_dec(mb)} Mb"
  else
    kb = num/1024.0
    mb = kb / 1024.0
    gb = mb / 1024.0
    return "#{f_to_dec(gb)} Gb"
  end
end
def f_to_dec(f, prec=2,sep='.')
  num = f.to_i.to_s
  dig = ((prec-(post=((f*(10**prec)).to_i%(10**prec)).to_s).size).times do post='0'+post end; post)
  return num+sep+dig
end


#############
##  START  ##
#############

## Execution Start here ##
def main_program
  
  check_directories
  
  check_answer
  
  ### START MYSQL DUMP ###
  puts @lines
  puts "Starting MYSQL Dump \n"
  sleep 1
  if @all_databases 
    puts "INFO: Going to dump all databases into"
    puts "  '#{@data_dir}'"
    check_answer
  else
    puts "INFO: Going to dump '#{@databases.join(", ")}' databases into"
    puts "  '#{@data_dir}'"
  end

  # Check if destination directory at local exists
  begin
    puts "INFO: Creating Directory '#{@data_dir}' with mode 700"
    FileUtils.mkdir_p @data_dir, :mode => 0700 
  rescue
    puts "WARNING: directory '#{@data_dir}' could not be created, check your permissions."
    puts "Going to use '/tmp' folder instead."
    @data_dir = "/tmp/#{@data_dir}"
  end
  
  puts @lines
  puts "Starting MYSQL dump?"
  check_answer
  make_mysql_backup
  puts "Done Dumping SQL data."
  
  puts @lines
  puts "Stablishing Connection to S3 account."
  stablish_connection
  
  find_or_create_bucket
  
  puts @lines
  puts "Now Going to copy Data to S3 bucket #{@bucket_name}."
  send_data
  
  puts @lines
  puts "#{@time} -- DONE"
  puts @lines

end

main_program

require 'aws/s3'

@lines = "\n----------------------------------------------------------"
@portage_name = "portage.tar.gz"
@bucket_name = "gentoo_portage"
@local_portage = "/mnt/portage.tar.gz"

@access_key_id = ENV['AMAZON_ACCESS_KEY_ID']
@secret_access_key = ENV['AMAZON_SECRET_ACCESS_KEY']

def update_portage
  puts "Updating portage via EIX"
  puts "Wait 2 minutes"
  Thread.new do
    IO.popen("eix-sync")
  end
  sleep 120
end

# Function to stablish connection
def stablish_connection
  begin
    AWS::S3::Base.establish_connection!(
      :access_key_id     => @access_key_id,
      :secret_access_key => @secret_access_key
    )
  rescue => exception
    puts @lines
    puts "There was an error: "
    puts exception.to_s
    exit
  end
  puts "Good, Connection Stablished."
end


# Function to find or create a bucket
def find_or_create_bucket
  begin
    AWS::S3::Bucket.find(@bucket_name)
  rescue
    puts "Bucket #{@bucket_name} not found."
    puts 'Creating the bucket now.'
    AWS::S3::Bucket.create(@bucket_name)
    puts "Good, bucket #{@bucket_name} created."
  end
  puts "Good, bucket #{@bucket_name} found."
end

# Function to make the Database Dumps
def tar_portage
  puts "Starting to make tar file of local portage. Wait 2 minutes"
  puts "cd /mnt/gentoo/ && rm -rf #{@local_portage} && tar -czpf #{@local_portage} portage"
  Thread.new do
    IO.popen("cd /mnt/gentoo/ && rm -rf #{@local_portage} && tar -czpf #{@local_portage} portage")
  end
  sleep 120
end


# Function to send data to bucket
def send_data
  # Store it
  puts "Starting data Transfer"
  AWS::S3::S3Object.store(@portage_name, open(@local_portage), @bucket_name)
  @data_transferred = File.size(@local_portage)
  puts "Data Transfered: #{to_file_size(@data_transferred)}"
  puts "done!"
end

# to print it nicely the data send
def to_file_size(num)
  case num
  when 0 
    return "0 byte"
  when 1..1024
    return "1K"
  when 1025..1048576
    kb = num/1024
    return "#{kb} Kb"
  when 1024577..1049165824
    kb = num/1024
    mb = kb / 1024
    return "#{mb} Mb"
  else
    kb = num/1024
    mb = kb / 1024
    gb = mb / 1024
    return "#{gb} Gb"
  end
end

update_portage
stablish_connection
find_or_create_bucket
tar_portage
send_data

require 'aws/s3'

@portage_name = "portage.tar.gz"
@bucket_name = "gentoo_portage"
@local_portage = "/mnt/portage.tar.gz"

@access_key_id = ENV['AMAZON_ACCESS_KEY_ID']
@secret_access_key = ENV['AMAZON_SECRET_ACCESS_KEY']

def download_portage
  AWS::S3::Base.establish_connection!( :access_key_id => @access_key_id, :secret_access_key => @secret_access_key)
  bucket =  AWS::S3::Bucket.find(@bucket_name)
  puts "Removing old portage"
  FileUtils.rm(@local_portage, :force => true)
  puts "Downloading new portage"
  open(@local_portage, 'w') do |file|
    AWS::S3::S3Object.stream(@portage_name, @bucket_name) do |chunk|
      file.write chunk
    end
  end
end

def untar_portage
  FileUtils.rm("/mnt/gentoo/portage", :force => true)
  puts "Starting to untar file to local portage. Wait 2 minutes"
  puts "cd /mnt/gentoo/ && tar -xzpf #{@local_portage}"
  Thread.new do
    IO.popen("cd /mnt/gentoo/ && rm -rf portage && tar -xzpf #{@local_portage}")
  end
  sleep 120
end

def update_portage
  puts "Updating portage via EIX"
  Thread.new do
    IO.popen("eix-sync")
  end
end

download_portage
untar_portage
update_portage