Commit e783bb9d authored by 神楽坂玲奈's avatar 神楽坂玲奈

fix

parent 9b40d56c
...@@ -213,12 +213,12 @@ begin ...@@ -213,12 +213,12 @@ begin
def batch_download(main_url, files, content_type=nil) def batch_download(main_url, files, content_type=nil)
connections = {} connections = {}
retried = [0] #pass by reference count = {total: files.size, error: 0}
[1*100, files.size].min.times { do_download(main_url, files, content_type, retried, connections) } [10*100, files.size].min.times { do_download(main_url, files, content_type, count, connections) }
end end
def do_download(main_url, files, content_type, retried, connections) def do_download(main_url, files, content_type, count, connections)
if connections.size < 1 if connections.size < 10
connection = EventMachine::HttpRequest.new(main_url) connection = EventMachine::HttpRequest.new(main_url)
connections[connection] = 0 connections[connection] = 0
else else
...@@ -232,15 +232,13 @@ begin ...@@ -232,15 +232,13 @@ begin
remote_path, local_path = files.shift remote_path, local_path = files.shift
connections[connection] += 1 connections[connection] += 1
connection.get(path: remote_path, keepalive: connections[connection] != 100).callback { |http| connection.get(path: remote_path, keepalive: connections[connection] != 100).callback { |http|
puts local_path puts File.basename local_path
retried[0] = 0 count[:error] = 0
count[:total] -= 1
if http.response_header['CONNECTION'] != 'keep-alive' if http.response_header['CONNECTION'] != 'keep-alive'
connection.close connection.close
connections.delete(connection) connections.delete(connection)
do_download(main_url, files, content_type, retried, connections) while !files.empty? and (connections.size < 1 or connections.values.min < 100) do_download(main_url, files, content_type, count, connections) while !files.empty? and (connections.size < 10 or connections.values.min < 100)
elsif files.empty?
connection.close
connections.delete(connection)
end end
if http.response_header.status == 200 and (!content_type or http.response_header['CONTENT_TYPE'] == content_type) if http.response_header.status == 200 and (!content_type or http.response_header['CONTENT_TYPE'] == content_type)
...@@ -248,18 +246,31 @@ begin ...@@ -248,18 +246,31 @@ begin
else else
puts http.response_header.http_status puts http.response_header.http_status
end end
if count[:total].zero?
connections.each_key { |connection| connection.close }
connections.clear
puts 'all done'
EM.stop
end
}.errback { |http| }.errback { |http|
puts http.error puts http.error
connection.close connection.close
connections.delete(connection) connections.delete(connection)
files[remote_path] = local_path files[remote_path] = local_path
retried[0] += 1 count[:error] += 1
if retried[0] <= 10*100 if count[:error] <= 10*100
do_download(main_url, files, content_type, retried, connections) while !files.empty? and (connections.size < 1 or connections.values.min < 100) do_download(main_url, files, content_type, count, connections) while !files.empty? and (connections.size < 10 or connections.values.min < 100)
else
connections.each_key { |connection| connection.close }
connections.clear
puts 'network error'
EM.stop
end end
} }
end end
def load_system_conf def load_system_conf
system_conf = {} system_conf = {}
conf_path = File.join(File.dirname(Config['ygopro']['path']), 'system.conf') conf_path = File.join(File.dirname(Config['ygopro']['path']), 'system.conf')
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment