Class | Gem::RemoteFetcher |
In: |
lib/rubygems/remote_fetcher.rb
|
Parent: | Object |
RemoteFetcher handles the details of fetching gems and gem information from a remote source.
Cached RemoteFetcher instance.
# File lib/rubygems/remote_fetcher.rb, line 40 40: def self.fetcher 41: @fetcher ||= self.new Gem.configuration[:http_proxy] 42: end
Initialize a remote fetcher using the source URI and possible proxy information.
proxy
variable setting
HTTP_PROXY_PASS)
# File lib/rubygems/remote_fetcher.rb, line 55 55: def initialize(proxy = nil) 56: require 'net/http' 57: require 'stringio' 58: require 'time' 59: require 'uri' 60: 61: Socket.do_not_reverse_lookup = true 62: 63: @connections = {} 64: @requests = Hash.new 0 65: @proxy_uri = 66: case proxy 67: when :no_proxy then nil 68: when nil then get_proxy_from_env 69: when URI::HTTP then proxy 70: else URI.parse(proxy) 71: end 72: @user_agent = user_agent 73: end
Creates or an HTTP connection based on uri, or retrieves an existing connection, using a proxy if needed.
# File lib/rubygems/remote_fetcher.rb, line 299 299: def connection_for(uri) 300: net_http_args = [uri.host, uri.port] 301: 302: if @proxy_uri then 303: net_http_args += [ 304: @proxy_uri.host, 305: @proxy_uri.port, 306: @proxy_uri.user, 307: @proxy_uri.password 308: ] 309: end 310: 311: connection_id = [Thread.current.object_id, *net_http_args].join ':' 312: @connections[connection_id] ||= Net::HTTP.new(*net_http_args) 313: connection = @connections[connection_id] 314: 315: if uri.scheme == 'https' and not connection.started? then 316: require 'net/https' 317: connection.use_ssl = true 318: connection.verify_mode = OpenSSL::SSL::VERIFY_NONE 319: end 320: 321: connection.start unless connection.started? 322: 323: connection 324: rescue Errno::EHOSTDOWN => e 325: raise FetchError.new(e.message, uri) 326: end
# File lib/rubygems/remote_fetcher.rb, line 328 328: def correct_for_windows_path(path) 329: if path[0].chr == '/' && path[1].chr =~ /[a-z]/i && path[2].chr == ':' 330: path = path[1..-1] 331: else 332: path 333: end 334: end
Moves the gem spec from source_uri to the cache dir unless it is already there. If the source_uri is local the gem cache dir copy is always replaced.
# File lib/rubygems/remote_fetcher.rb, line 98 98: def download(spec, source_uri, install_dir = Gem.dir) 99: Gem.ensure_gem_subdirectories(install_dir) rescue nil 100: 101: if File.writable?(install_dir) 102: cache_dir = File.join install_dir, "cache" 103: else 104: cache_dir = File.join Gem.user_dir, "cache" 105: end 106: 107: gem_file_name = File.basename spec.cache_file 108: local_gem_path = File.join cache_dir, gem_file_name 109: 110: FileUtils.mkdir_p cache_dir rescue nil unless File.exist? cache_dir 111: 112: # Always escape URI's to deal with potential spaces and such 113: unless URI::Generic === source_uri 114: source_uri = URI.parse(URI.const_defined?(:DEFAULT_PARSER) ? 115: URI::DEFAULT_PARSER.escape(source_uri.to_s) : 116: URI.escape(source_uri.to_s)) 117: end 118: 119: scheme = source_uri.scheme 120: 121: # URI.parse gets confused by MS Windows paths with forward slashes. 122: scheme = nil if scheme =~ /^[a-z]$/i 123: 124: case scheme 125: when 'http', 'https' then 126: unless File.exist? local_gem_path then 127: begin 128: say "Downloading gem #{gem_file_name}" if 129: Gem.configuration.really_verbose 130: 131: remote_gem_path = source_uri + "gems/#{gem_file_name}" 132: 133: gem = self.fetch_path remote_gem_path 134: rescue Gem::RemoteFetcher::FetchError 135: raise if spec.original_platform == spec.platform 136: 137: alternate_name = "#{spec.original_name}.gem" 138: 139: say "Failed, downloading gem #{alternate_name}" if 140: Gem.configuration.really_verbose 141: 142: remote_gem_path = source_uri + "gems/#{alternate_name}" 143: 144: gem = self.fetch_path remote_gem_path 145: end 146: 147: File.open local_gem_path, 'wb' do |fp| 148: fp.write gem 149: end 150: end 151: when 'file' then 152: begin 153: path = source_uri.path 154: path = File.dirname(path) if File.extname(path) == '.gem' 155: 156: remote_gem_path = correct_for_windows_path(File.join(path, 'gems', gem_file_name)) 157: 158: FileUtils.cp(remote_gem_path, local_gem_path) 159: rescue Errno::EACCES 160: local_gem_path = source_uri.to_s 161: end 162: 163: say "Using local gem #{local_gem_path}" if 164: Gem.configuration.really_verbose 165: when nil then # TODO test for local overriding cache 166: source_path = if Gem.win_platform? && source_uri.scheme && 167: !source_uri.path.include?(':') then 168: "#{source_uri.scheme}:#{source_uri.path}" 169: else 170: source_uri.path 171: end 172: 173: source_path = unescape source_path 174: 175: begin 176: FileUtils.cp source_path, local_gem_path unless 177: File.expand_path(source_path) == File.expand_path(local_gem_path) 178: rescue Errno::EACCES 179: local_gem_path = source_uri.to_s 180: end 181: 182: say "Using local gem #{local_gem_path}" if 183: Gem.configuration.really_verbose 184: else 185: raise Gem::InstallError, "unsupported URI scheme #{source_uri.scheme}" 186: end 187: 188: local_gem_path 189: end
Given a name and requirement, downloads this gem into cache and returns the filename. Returns nil if the gem cannot be located.
# File lib/rubygems/remote_fetcher.rb, line 82 82: def download_to_cache dependency 83: found = Gem::SpecFetcher.fetcher.fetch dependency, true, true, 84: dependency.prerelease? 85: 86: return if found.empty? 87: 88: spec, source_uri = found.sort_by { |(s,_)| s.version }.last 89: 90: download spec, source_uri 91: end
# File lib/rubygems/remote_fetcher.rb, line 251 251: def escape(str) 252: return unless str 253: @uri_parser ||= uri_escaper 254: @uri_parser.escape str 255: end
File Fetcher. Dispatched by fetch_path. Use it instead.
# File lib/rubygems/remote_fetcher.rb, line 194 194: def fetch_file uri, *_ 195: Gem.read_binary correct_for_windows_path uri.path 196: end
HTTP Fetcher. Dispatched by fetch_path. Use it instead.
# File lib/rubygems/remote_fetcher.rb, line 201 201: def fetch_http uri, last_modified = nil, head = false, depth = 0 202: fetch_type = head ? Net::HTTP::Head : Net::HTTP::Get 203: response = request uri, fetch_type, last_modified 204: 205: case response 206: when Net::HTTPOK, Net::HTTPNotModified then 207: head ? response : response.body 208: when Net::HTTPMovedPermanently, Net::HTTPFound, Net::HTTPSeeOther, 209: Net::HTTPTemporaryRedirect then 210: raise FetchError.new('too many redirects', uri) if depth > 10 211: 212: location = URI.parse response['Location'] 213: fetch_http(location, last_modified, head, depth + 1) 214: else 215: raise FetchError.new("bad response #{response.message} #{response.code}", uri) 216: end 217: end
Downloads uri and returns it as a String.
# File lib/rubygems/remote_fetcher.rb, line 224 224: def fetch_path(uri, mtime = nil, head = false) 225: uri = URI.parse uri unless URI::Generic === uri 226: 227: raise ArgumentError, "bad uri: #{uri}" unless uri 228: raise ArgumentError, "uri scheme is invalid: #{uri.scheme.inspect}" unless 229: uri.scheme 230: 231: data = send "fetch_#{uri.scheme}", uri, mtime, head 232: data = Gem.gunzip data if data and not head and uri.to_s =~ /gz$/ 233: data 234: rescue FetchError 235: raise 236: rescue Timeout::Error 237: raise FetchError.new('timed out', uri.to_s) 238: rescue IOError, SocketError, SystemCallError => e 239: raise FetchError.new("#{e.class}: #{e}", uri.to_s) 240: end
Returns the size of uri in bytes.
# File lib/rubygems/remote_fetcher.rb, line 245 245: def fetch_size(uri) # TODO: phase this out 246: response = fetch_path(uri, nil, true) 247: 248: response['content-length'].to_i 249: end
Returns an HTTP proxy URI if one is set in the environment variables.
# File lib/rubygems/remote_fetcher.rb, line 272 272: def get_proxy_from_env 273: env_proxy = ENV['http_proxy'] || ENV['HTTP_PROXY'] 274: 275: return nil if env_proxy.nil? or env_proxy.empty? 276: 277: uri = URI.parse(normalize_uri(env_proxy)) 278: 279: if uri and uri.user.nil? and uri.password.nil? then 280: # Probably we have http_proxy_* variables? 281: uri.user = escape(ENV['http_proxy_user'] || ENV['HTTP_PROXY_USER']) 282: uri.password = escape(ENV['http_proxy_pass'] || ENV['HTTP_PROXY_PASS']) 283: end 284: 285: uri 286: end
Normalize the URI by adding "http://" if it is missing.
# File lib/rubygems/remote_fetcher.rb, line 291 291: def normalize_uri(uri) 292: (uri =~ /^(https?|ftp|file):/) ? uri : "http://#{uri}" 293: end
Read the data from the (source based) URI, but if it is a file:// URI, read from the filesystem instead.
# File lib/rubygems/remote_fetcher.rb, line 340 340: def open_uri_or_path(uri, last_modified = nil, head = false, depth = 0) 341: raise "NO: Use fetch_path instead" 342: # TODO: deprecate for fetch_path 343: end
Performs a Net::HTTP request of type request_class on uri returning a Net::HTTP response object. request maintains a table of persistent connections to reduce connect overhead.
# File lib/rubygems/remote_fetcher.rb, line 350 350: def request(uri, request_class, last_modified = nil) 351: request = request_class.new uri.request_uri 352: 353: unless uri.nil? || uri.user.nil? || uri.user.empty? then 354: request.basic_auth uri.user, uri.password 355: end 356: 357: request.add_field 'User-Agent', @user_agent 358: request.add_field 'Connection', 'keep-alive' 359: request.add_field 'Keep-Alive', '30' 360: 361: if last_modified then 362: last_modified = last_modified.utc 363: request.add_field 'If-Modified-Since', last_modified.rfc2822 364: end 365: 366: yield request if block_given? 367: 368: connection = connection_for uri 369: 370: retried = false 371: bad_response = false 372: 373: begin 374: @requests[connection.object_id] += 1 375: 376: say "#{request.method} #{uri}" if 377: Gem.configuration.really_verbose 378: 379: file_name = File.basename(uri.path) 380: # perform download progress reporter only for gems 381: if request.response_body_permitted? && file_name =~ /\.gem$/ 382: reporter = ui.download_reporter 383: response = connection.request(request) do |incomplete_response| 384: if Net::HTTPOK === incomplete_response 385: reporter.fetch(file_name, incomplete_response.content_length) 386: downloaded = 0 387: data = '' 388: 389: incomplete_response.read_body do |segment| 390: data << segment 391: downloaded += segment.length 392: reporter.update(downloaded) 393: end 394: reporter.done 395: if incomplete_response.respond_to? :body= 396: incomplete_response.body = data 397: else 398: incomplete_response.instance_variable_set(:@body, data) 399: end 400: end 401: end 402: else 403: response = connection.request request 404: end 405: 406: say "#{response.code} #{response.message}" if 407: Gem.configuration.really_verbose 408: 409: rescue Net::HTTPBadResponse 410: say "bad response" if Gem.configuration.really_verbose 411: 412: reset connection 413: 414: raise FetchError.new('too many bad responses', uri) if bad_response 415: 416: bad_response = true 417: retry 418: # HACK work around EOFError bug in Net::HTTP 419: # NOTE Errno::ECONNABORTED raised a lot on Windows, and make impossible 420: # to install gems. 421: rescue EOFError, Timeout::Error, 422: Errno::ECONNABORTED, Errno::ECONNRESET, Errno::EPIPE 423: 424: requests = @requests[connection.object_id] 425: say "connection reset after #{requests} requests, retrying" if 426: Gem.configuration.really_verbose 427: 428: raise FetchError.new('too many connection resets', uri) if retried 429: 430: reset connection 431: 432: retried = true 433: retry 434: end 435: 436: response 437: end
Resets HTTP connection connection.
# File lib/rubygems/remote_fetcher.rb, line 442 442: def reset(connection) 443: @requests.delete connection.object_id 444: 445: connection.finish 446: connection.start 447: end
# File lib/rubygems/remote_fetcher.rb, line 257 257: def unescape(str) 258: return unless str 259: @uri_parser ||= uri_escaper 260: @uri_parser.unescape str 261: end
# File lib/rubygems/remote_fetcher.rb, line 263 263: def uri_escaper 264: URI::Parser.new 265: rescue NameError 266: URI 267: end
# File lib/rubygems/remote_fetcher.rb, line 449 449: def user_agent 450: ua = "RubyGems/#{Gem::VERSION} #{Gem::Platform.local}" 451: 452: ruby_version = RUBY_VERSION 453: ruby_version += 'dev' if RUBY_PATCHLEVEL == -1 454: 455: ua << " Ruby/#{ruby_version} (#{RUBY_RELEASE_DATE}" 456: if RUBY_PATCHLEVEL >= 0 then 457: ua << " patchlevel #{RUBY_PATCHLEVEL}" 458: elsif defined?(RUBY_REVISION) then 459: ua << " revision #{RUBY_REVISION}" 460: end 461: ua << ")" 462: 463: ua << " #{RUBY_ENGINE}" if defined?(RUBY_ENGINE) and RUBY_ENGINE != 'ruby' 464: 465: ua 466: end