Upload and Download Gentoo portage to S3

Posted on December 07, 2007 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99


require 'aws/s3'

@lines = "\n----------------------------------------------------------"
@portage_name = "portage.tar.gz"
@bucket_name = "gentoo_portage"
@local_portage = "/mnt/portage.tar.gz"

@access_key_id = ENV['AMAZON_ACCESS_KEY_ID']
@secret_access_key = ENV['AMAZON_SECRET_ACCESS_KEY']

def update_portage
  puts "Updating portage via EIX"
  puts "Wait 2 minutes"
  Thread.new do
    IO.popen("eix-sync")
  end
  sleep 120
end

# Function to stablish connection
def stablish_connection
  begin
    AWS::S3::Base.establish_connection!(
      :access_key_id     => @access_key_id,
      :secret_access_key => @secret_access_key
    )
  rescue => exception
    puts @lines
    puts "There was an error: "
    puts exception.to_s
    exit
  end
  puts "Good, Connection Stablished."
end


# Function to find or create a bucket
def find_or_create_bucket
  begin
    AWS::S3::Bucket.find(@bucket_name)
  rescue
    puts "Bucket #{@bucket_name} not found."
    puts 'Creating the bucket now.'
    AWS::S3::Bucket.create(@bucket_name)
    puts "Good, bucket #{@bucket_name} created."
  end
  puts "Good, bucket #{@bucket_name} found."
end

# Function to make the Database Dumps
def tar_portage
  puts "Starting to make tar file of local portage. Wait 2 minutes"
  puts "cd /mnt/gentoo/ && rm -rf #{@local_portage} && tar -czpf #{@local_portage} portage"
  Thread.new do
    IO.popen("cd /mnt/gentoo/ && rm -rf #{@local_portage} && tar -czpf #{@local_portage} portage")
  end
  sleep 120
end


# Function to send data to bucket
def send_data
  # Store it
  puts "Starting data Transfer"
  AWS::S3::S3Object.store(@portage_name, open(@local_portage), @bucket_name)
  @data_transferred = File.size(@local_portage)
  puts "Data Transfered: #{to_file_size(@data_transferred)}"
  puts "done!"
end

# to print it nicely the data send
def to_file_size(num)
  case num
  when 0 
    return "0 byte"
  when 1..1024
    return "1K"
  when 1025..1048576
    kb = num/1024
    return "#{kb} Kb"
  when 1024577..1049165824
    kb = num/1024
    mb = kb / 1024
    return "#{mb} Mb"
  else
    kb = num/1024
    mb = kb / 1024
    gb = mb / 1024
    return "#{gb} Gb"
  end
end

update_portage
stablish_connection
find_or_create_bucket
tar_portage
send_data

To get it back 

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45



require 'aws/s3'

@portage_name = "portage.tar.gz"
@bucket_name = "gentoo_portage"
@local_portage = "/mnt/portage.tar.gz"

@access_key_id = ENV['AMAZON_ACCESS_KEY_ID']
@secret_access_key = ENV['AMAZON_SECRET_ACCESS_KEY']

def download_portage
  AWS::S3::Base.establish_connection!( :access_key_id => @access_key_id, :secret_access_key => @secret_access_key)
  bucket =  AWS::S3::Bucket.find(@bucket_name)
  puts "Removing old portage"
  FileUtils.rm(@local_portage, :force => true)
  puts "Downloading new portage"
  open(@local_portage, 'w') do |file|
    AWS::S3::S3Object.stream(@portage_name, @bucket_name) do |chunk|
      file.write chunk
    end
  end
end

def untar_portage
  FileUtils.rm("/mnt/gentoo/portage", :force => true)
  puts "Starting to untar file to local portage. Wait 2 minutes"
  puts "cd /mnt/gentoo/ && tar -xzpf #{@local_portage}"
  Thread.new do
    IO.popen("cd /mnt/gentoo/ && rm -rf portage && tar -xzpf #{@local_portage}")
  end
  sleep 120
end

def update_portage
  puts "Updating portage via EIX"
  Thread.new do
    IO.popen("eix-sync")
  end
end

download_portage
untar_portage
update_portage

Tagged with:         |