Procesando Millones de Imágenes Ruby vs Elixir

8/17/2019 Procesando Millones de Imágenes Ruby vs Elixir

http://slidepdf.com/reader/full/procesando-millones-de-imagenes-ruby-vs-elixir 1/75

Procesando millones de

imágenesDavid Padilla

@dabit



Carrierwave



image.recreate_versions!



Image.each do |i| i.file.recreate_versions!end



2,700,000



1 segundo



2,700,000 ÷ 60

=

45,000 minutos



45,000 ÷ 60

=

750 horas



750 ÷ 24

=

31.25 días



MAX_WORKERS = 20

Image.find_each(batch_size: 1200) do |batch| queue = Queue.new batch.each {|o| queue.push o} workers = (0..MAX_WORKERS).map do | w|

Thread.new do while image = queue.pop(true) image.file.recreate_versions! end end

end

workers.map(&:join)end



MAX_WORKERS = 20

Image.find_each(batch_size: 1200) do |batch| queue = Queue.new batch.each {|o| queue.push o} workers = (0..MAX_WORKERS).map do |w|


end




MAX_WORKERS = 20



end




MAX_WORKERS = 20

Image.find_each(batch_size: 1200) do |batch| queue = Queue.new batch.each {|o| queue.push o} workers = (0..MAX_WORKERS).map do | w|


end




array = []

5.times.map do Thread.new do 1000.times do array << nil end end

end.each(&:join)

puts array.size



$ ruby pushing_nil.rb5000

$ jruby pushing_nil.rb4446

$ rbx pushing_nil.rb3088



array = []

5.times.map do Thread.new do 1000.times do array << nil end end

end.each(&:join)

puts array.size



1 segundo



0.6 segundos



18 días



Elixir



class Code def method(*args) code = code + code

while(true) do something_awesome end end

def something_awesome add = 1 + 2 o = Object.new o.save_to_database endend



Exception



Traer registros de la Base de Datos

Bajar de S3 imagen original

Crear dos tamaños

Subir a S3

Crear aplicación



Crear aplicación



$ mix new images



Traer registros de la Basede Datos



use Mix.Config

config :images, Images.Repo,

adapter: Ecto. Adapters. MySQL, database: "images_database", username: "root", password: "",

hostname: "localhost"



defmodule Images.PropertyImage do use Ecto. Model

schema "images" do field :file, :string end

end



defmodule Images.Image do use Ecto. Model

def main_query do

from i in Images.Image, select: i end def find(image_id ) do

from i in main_query

where: id = ^image_id end

def paged(offset, limit) do from i in main_query, limit: ^limit,

offset: ^offset end def all do

Images.Repo.all main_query end

end



Bajar de S3 la imagenoriginal



def download_original(filename , id ) do file = temp_filename(filename, id) ibrowse = [save_response_to_file: String.to_char_list(file)] s3_url(filename, id)

|> HTTPotion.get([ibrowse: ibrowse]) fileend

def s3_url(file , id ) do "#{s3_path}/#{id}/#{file}"end



def download_original(filename , id ) do file = temp_filename(filename, id) ibrowse = [save_response_to_file: String.to_char_list(file)]HTTPotion.get(s3_url(filename, id), [ibrowse: ibrowse])

fileend




Crear dos tamaños



def generate_medium (file , filename , id ) do result = Path.join(System .tmp_dir, size_name(filename, :medium ))

Mogrify.open(file) |> Mogrify.copy |> Mogrify.resize_to_fill("450x300") |> Mogrify.save(result)end



Subir a S3



{status, _} = System .cmd("s3cmd", ["-P", "put", filename, s3_name])



{_, file} = File.read(result)

Application.get_env(:images, :s3_bucket) |> String.to_char_list |> :erlcloud_s3.put_object(s3_name, file, [], [{#…}])



y la concurrencia?



Traer registros de la Base de Datos

Bajar de S3 imagen original

Crear dos tamaños

Subir a S3



:poolboy



defmodule Images.PropertyImageWorker do use GenServer

def start_link([]) do :gen_server.start_link( __MODULE__ , [], []) end

def init(state ) do secret = to_char_list( Application.get_env(:images, :aws_secret_key))

Application.get_env(:images, :aws_access_key) |> to_char_list |> :erlcloud_s3.configure(secret) {:ok, state} end

def handle_call(image , from , state ) do

result = Images.PropertyImage.process(image) {:reply, [result], state} endend



defmodule Images.PropertyImageWorker do use GenServer

def start_link([]) do :gen_server.start_link(__MODULE__, [], []) end

def init(state ) do secret = to_char_list( Application.get_env(:images, :aws_secret_key))

Application.get_env(:images, :aws_access_key) |> to_char_list |> :erlcloud_s3.configure(secret) {:ok, state} end

def handle_call(image , from , state ) do

result = Images.PropertyImage.process(image) {:reply, [result], state} endend



defmodule Images.ImagesSupervisor do use Supervisor

def start_link do :supervisor.start_link( __MODULE__ , [])

end

def init([]) do poolboy_config = [ {:name, {:local, pool_name()}}, {:worker_module, Images.PropertyImageWorker}, {:size, 20}, {:max_overflow , 0} ]

children = [ :poolboy.child_spec(pool_name(), poolboy_config, []), worker(Images.Repo, []) ]

supervise(children, strategy: :one_for_one) end

def pool_name do :property_images endend



defmodule Images.ImagesSupervisor do use Supervisor

def start_link do :supervisor.start_link(__MODULE__, [])

end

def init([]) do poolboy_config = [ {:name, {:local, pool_name()}}, {:worker_module, Images.PropertyImageWorker}, {:size, 20}, {:max_overflow , 0} ]

children = [ :poolboy.child_spec(pool_name(), poolboy_config, []), worker(Images.Repo, []) ]

supervise(children, strategy: :one_for_one) end

def pool_name do :property_images endend



defmodule Images do def start( _type , _args) do

supervisor = Images.ImagesSupervisor.start_link enqueue supervisor end

def enqueue do Images.PropertyImage.all

|> Enum .each fn(r ) -> spawn(fn() -> pool_image(r) end) end end

def pool_image(image ) do :poolboy.transaction( Images.ImagesSupervisor.pool_name, fn( pid ) -> :gen_server.call(pid, image) end,

:infinity ) endend








:infinity ) endend



BEAM



Conclusión



4 días



4 x 24 = 96

96 x 60 = 5,7605,760 x 60 = 345,600

345,600 / 2,700,000 = 0.128s



Conclusión II



Fin

Procesando Millones de Imágenes Ruby vs Elixir

Documents

Transcript of Procesando Millones de Imágenes Ruby vs Elixir