Joey Marino
4/2/2008 4:53:00 PM
[Note: parts of this message were removed to make it a legal post.]
Ok, I was able to get it all into one class. The problem lies in this class:
class Picture
def initialize(db,rets,rets_class)
@db = db
@rets = rets
@rets_class = rets_class
@attempts = 0
end
def getPic(key)
begin
get_object_request = GetObjectRequest.new(@rets_class, "Photo")
get_object_request.add_all_objects(key)
get_object_response = @rets.session.get_object(get_object_request)
content_type_suffixes = { "image/jpeg" => "jpg"}
makePicDir(key)
get_object_response.each_object do |object_descriptor|
object_key = object_descriptor.object_key
obj_id = object_descriptor.object_id
content_type = object_descriptor.content_type
description = object_descriptor.description
#print "#{object_key} object \##{object_id}"
#print ", description: #{description}" if !description.empty?
#puts
suffix = content_type_suffixes[content_type]
pic = object_descriptor.data_as_string
savePic(key,obj_id.to_s,suffix,description,pic)
end
get_object_response = nil
rescue => e
puts "Error retrieving pictures for #{key}: " + e
if @attempts <= 5
@attempts += 1
puts "retrying"
retry
else
puts "failed"
@attempts = 0
end
end
@attempts = 0
end
def getThumb(key)
begin
get_object_request = GetObjectRequest.new(@rets_class, "Thumbnail")
get_object_request.add_all_objects(key)
get_object_response = @rets.session.get_object(get_object_request)
content_type_suffixes = { "image/jpeg" => "jpg"}
get_object_response.each_object do |object_descriptor|
object_key = object_descriptor.object_key
obj_id = object_descriptor.object_id
content_type = object_descriptor.content_type
description = object_descriptor.description
#print "#{object_key} object \##{object_id}"
#print ", description: #{description}" if !description.empty?
#puts
suffix = content_type_suffixes[content_type]
pic = object_descriptor.data_as_string
savePic(key,obj_id.to_s,suffix,description,pic,true)
end
get_object_response = nil
rescue => e
puts "Error retrieving thumbs for #{key}: " + e
if @attempts <= 5
@attempts += 1
puts "retrying"
retry
else
puts "failed"
@attempts = 0
end
end
@attempts = 0
end
def makePicDir(key)
FileUtils.mkpath("#{$pic_dir}#{key}/thumb")
end
def savePic(key,id,suffix,desc,pic,thumb_bool=false)
if thumb_bool
file_name = $pic_dir + key + "/thumb/" + id + "." + suffix
location = "/" + key + "/thumb/" + id + "." + suffix
else
file_name = $pic_dir + key + "/" + id + "." + suffix
location = "/" + key + "/" + id + "." + suffix
end
self.savePicFile(file_name,pic)
size = File.size(file_name)
if thumb_bool
self.insertThumbDB(key,id,location)
else
self.insertPicDB(key,id,desc,size,location)
end
end
def savePicFile(file_name,pic)
f = File.open(file_name, "wb")
f << pic
f.close
end
def insertPicDB(key,id,desc,size,location)
description = @db.database.escape_string(desc)
if
@db.DBinsert("PICS","pkey,id,description,size,location","#{key},#{id},'#{description}','#{size}','#{location}'")
# puts "#{key} #{id} pic added"
print ":"
end
end
def insertThumbDB(key,id,location)
if @db.DBupdate("PICS","thumb = '#{location}'"," pkey = #{key} and id =
#{id}")
# puts "#{key} #{id} thumb added"
print "."
end
end
def deletePic(key)
self.deletePicDir(key)
self.deletePicDB(key)
end
def deletePicDir(key)
if File.exists?("#{$pic_dir}#{key}")
FileUtils.remove_dir("#{$pic_dir}#{key}")
end
end
def deletePicDB(key)
if @db.DBdelete("PICS","pkey = #{key}")
print "-"
# puts "#{key} pics deleted from db"
end
end
end #end class
On Wed, Apr 2, 2008 at 6:02 AM, Michal Suchanek <hramrach@centrum.cz> wrote:
> On 31/03/2008, Tim Hunter <TimHunter@nc.rr.com> wrote:
> > Joey Marino wrote:
> >
> > > I found out it is not the 3rd party code. I am downloading images
> > (~320,000
> > > ct.) as strings, and saving each to a file. The file handlers are
> getting
> > > closed. Using the Dike gem, I think I determined the bulk of the
> leaked
> > > memory is in these string variables. But what I don't understand is
> that
> > the
> > > variables containing the strings are local to a method of an object
> and
> > > should be unreferenced when that method is done, right? I've invoked
> GC at
> > > the end of the method and after each iteration that the method gets
> > called.
> > > I've even set the strings to nil after they've been saved to a file
> and
> > > before gc is called. The app is using 2G of ram and 4G of swap before
> it
> > > runs of out memory and crashes about 1/3rd of the way through. I'm
> really
> > > starting to doubt Ruby's ability to do memory intensive work. Any
> ideas?
> > >
> >
> > Many people have used Ruby for long-running tasks that use a lot of
> memory.
> > If Ruby was not collecting unused strings, don't you think somebody
> would
> > have noticed it by now?
>
> I have noticed :->
>
> Michal
>
>
--
Joey Marino