offtopic/open-pod
4
0
Fork 0
Code Issues 3 Pull requests Releases Wiki Activity

refactor: improve readability

Browse source
This commit is contained in:
danilo silva 2020-05-26 18:53:59 +00:00
parent 11d075c433
commit f74322d4d6
3 changed files with 118 additions and 82 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

151
lib/podcast_feed/provider/archive/parser.ex
View file

@ -1,68 +1,43 @@
defmodule PodcastFeed.Provider.Archive.Parser do defmodule PodcastFeed.Provider.Archive.Parser do
alias PodcastFeed.Utility.Format alias PodcastFeed.Utility.Format
alias __MODULE__
@extra_metadata_url "https://archive.org/download/{identifier}/metadata.json" @custom_metadata_url "https://archive.org/download/{identifier}/metadata.json"
@archive_metadata_url "http://archive.org/metadata/{identifier}" @archive_metadata_url "http://archive.org/metadata/{identifier}"
@download_url "https://archive.org/download/{identifier}/{filename}" @download_url "https://archive.org/download/{identifier}/{filename}"
@podcast_link "https://archive.org/details/{identifier}"
@extra_metadata_defaults %{ @custom_metadata_defaults %{
"link" => "", "link" => nil,
"image" => %{ "image" => %{
"url" => "", "url" => nil,
"title" => "", "title" => nil,
"link" => "", "link" => nil,
}, },
"category" => "", "category" => "",
"explicit" => "", "explicit" => "no",
"version" => "1",
} }
@enforce_keys [:identifier]
defstruct [:identifier, :podcast_data, :archive_metadata, custom_metadata: @custom_metadata_defaults]
def by_identifier(identifier) do def by_identifier(identifier) do
extra_metadata_json = fetch_extra_metadata(identifier) %Parser{identifier: identifier}
metadata_json = fetch_archive_metadata(identifier) |> enrich_with_archive_metadata()
parse(identifier, metadata_json, extra_metadata_json) |> enrich_with_custom_metadata()
|> to_podcast_feed_data()
end end
defp fetch_extra_metadata(identifier) do def to_podcast_feed_data(token) do
extra_metadata_url = Format.compile(@extra_metadata_url, identifier: identifier) %{
parse_extra_metadata_response(:hackney.get(extra_metadata_url, [], "", [follow_redirect: true])) podcast: podcast_data(token),
items: items_data(token)
}
end end
defp parse_extra_metadata_response({:ok, 200, _headers, client_ref}) do defp podcast_data(token = %{custom_metadata: custom, archive_metadata: %{"metadata" => metadata, "item_last_updated" => last_updated}}) do
{:ok, extra_metadata_json} = :hackney.body(client_ref) link = Format.compile(@podcast_link, identifier: token.identifier)
extra_metadata_json
|> String.split("\n")
|> Enum.join()
|> Poison.decode!()
end
defp parse_extra_metadata_response(_), do: @extra_metadata_defaults
defp fetch_archive_metadata(identifier) do
metadata_url = Format.compile(@archive_metadata_url, identifier: identifier)
{:ok, 200, _headers, client_ref} = :hackney.get(metadata_url, [], "", [follow_redirect: true, connect_timeout: 30000, recv_timeout: 30000])
{:ok, metadata_json} = :hackney.body(client_ref)
metadata_json |> Poison.decode!()
end
def parse(identifier, metadata = %{"files" => files}, extra) do
extra = files
|> fetch_cover(identifier)
|> enrich_extra_metadata_with_cover(extra)
%{podcast: podcast_data(metadata, extra), items: items_data(metadata, identifier)}
end
# cover is nil
defp enrich_extra_metadata_with_cover(nil, extra), do: extra
# cover is found and image is missing in the extra_metadata
defp enrich_extra_metadata_with_cover(cover, extra = %{"image" => %{"url" => ""}}) do
put_in(extra, ["image", "url"], cover)
end
# image is already set in the extra_metadata
defp enrich_extra_metadata_with_cover(_cover, extra), do: extra
defp podcast_data(%{"metadata" => metadata, "item_last_updated" => last_updated}, extra) do
%{ %{
title: metadata["title"], title: metadata["title"],
description: metadata["description"], description: metadata["description"],
@ -78,28 +53,48 @@ defmodule PodcastFeed.Provider.Archive.Parser do
author: metadata["creator"], author: metadata["creator"],
language: metadata["language"], language: metadata["language"],
image: %{ image: %{
url: extra["image"]["url"], url: get_in(custom, ["image", "url"]) || fetch_cover(token),
title: extra["image"]["title"], title: get_in(custom, ["image", "title"]) || metadata["title"],
link: extra["image"]["link"], link: get_in(custom, ["image", "link"]) || link,
}, },
link: extra["link"], link: Map.get(custom, "link") || link,
category: extra["category"], category: Map.get(custom, "category", ""),
explicit: extra["explicit"], explicit: Map.get(custom, "explicit", "no"),
} }
end end
defp items_data(%{identifier: identifier, archive_metadata: %{"files" => files}}) do
defp items_data(%{"files" => files}, identifier) do
files files
|> filter_audio_files() |> filter_audio_files()
|> Enum.map(fn f -> to_feed_item(f, identifier, files) end) |> Enum.map(fn f -> to_feed_item(f, identifier, files) end)
end end
defp fetch_custom_metadata(identifier) do
custom_metadata_url = Format.compile(@custom_metadata_url, identifier: identifier)
parse_custom_metadata_response(:hackney.get(custom_metadata_url, [], "", [follow_redirect: true]))
end
defp parse_custom_metadata_response({:ok, 200, _headers, client_ref}) do
{:ok, custom_metadata_json} = :hackney.body(client_ref)
custom_metadata_json
|> String.split("\n")
|> Enum.join()
|> Poison.decode!()
end
defp parse_custom_metadata_response(_), do: @custom_metadata_defaults
defp fetch_archive_metadata(identifier) do
metadata_url = Format.compile(@archive_metadata_url, identifier: identifier)
{:ok, 200, _headers, client_ref} = :hackney.get(metadata_url, [], "", [follow_redirect: true, connect_timeout: 30000, recv_timeout: 30000])
{:ok, metadata_json} = :hackney.body(client_ref)
metadata_json |> Poison.decode!()
end
defp filter_audio_files(files) do defp filter_audio_files(files) do
files |> Enum.filter(fn f -> Map.get(f, "format") =~ ~r/MP3/i end) #FIXME:! mp3, ogg, boh files |> Enum.filter(fn f -> Map.get(f, "format") =~ ~r/MP3/i end) #FIXME:! mp3, ogg, boh
end end
defp to_feed_item(file, identifier, files) do defp to_feed_item(file, identifier, _files) do
filename = Map.get(file, "name") filename = Map.get(file, "name")
%{ %{
title: file["title"], title: file["title"],
@ -109,16 +104,17 @@ defmodule PodcastFeed.Provider.Archive.Parser do
length: (file |> Map.get("length") |> Float.parse() |> elem(0)) |> trunc(), length: (file |> Map.get("length") |> Float.parse() |> elem(0)) |> trunc(),
size: file |> Map.get("size"), size: file |> Map.get("size"),
summary: "", summary: "",
image: download_url(identifier, fetch_image_of_audio(filename, files)), # image: download_url(identifier, fetch_image_of_audio(filename, files)),
image: nil,
keywords: file |> Map.take(["album", "artist", "genre"]) |> Map.values(), keywords: file |> Map.take(["album", "artist", "genre"]) |> Map.values(),
explicit: "no", explicit: "no",
} }
end end
defp fetch_cover(files, identifier) do defp fetch_cover(%{identifier: identifier, archive_metadata: %{"files" => files}}) do
filename = files filename = files
|> Enum.filter(fn f -> f["source"] == "original" end) |> Enum.filter(fn f -> f["source"] == "original" end)
|> Enum.filter(fn f -> f["format"] == "JPEG" end) #FIXME:! jpg, png, gif |> Enum.filter(fn f -> f["format"] =~ ~r/JPG|JPEG|PNG|GIF/i end)
|> List.first() |> List.first()
|> case do |> case do
nil -> nil nil -> nil
@ -128,22 +124,31 @@ defmodule PodcastFeed.Provider.Archive.Parser do
download_url(identifier, filename) download_url(identifier, filename)
end end
defp fetch_image_of_audio(audio_file, files) do # defp fetch_image_of_audio(audio_file, files) do
files # files
|> Enum.filter(fn # |> Enum.filter(fn
%{"format" => format, "source" => "derivative", "original" => ^audio_file} -> # %{"format" => format, "source" => "derivative", "original" => ^audio_file} ->
format =~ ~r/JPG|JPEG|PNG|GIF/i # format =~ ~r/JPG|JPEG|PNG|GIF/i
_ -> nil # _ -> nil
end) # end)
|> fetch_image_of_audio() # |> fetch_image_of_audio()
end # end
defp fetch_image_of_audio(image_files) when is_list(image_files), do: fetch_image_of_audio(List.first(image_files)) # defp fetch_image_of_audio(image_files) when is_list(image_files), do: fetch_image_of_audio(List.first(image_files))
defp fetch_image_of_audio(nil), do: nil # defp fetch_image_of_audio(nil), do: nil
defp fetch_image_of_audio(image_file), do: image_file |> Map.get("name", nil) # defp fetch_image_of_audio(image_file), do: image_file |> Map.get("name", nil)
defp download_url(_identifier, nil), do: nil defp download_url(_identifier, nil), do: nil
defp download_url(identifier, filename) do defp download_url(identifier, filename) do
Format.compile(@download_url, identifier: identifier, filename: filename) |> URI.encode() Format.compile(@download_url, identifier: identifier, filename: filename) |> URI.encode()
end end
defp enrich_with_archive_metadata(token) do
%Parser{ token | archive_metadata: fetch_archive_metadata(token.identifier) }
end
defp enrich_with_custom_metadata(token) do
%Parser{ token | custom_metadata: fetch_custom_metadata(token.identifier) }
end
end end

4
lib/podcast_feed_web/templates/feed/feed.xml.eex
View file

@ -41,9 +41,7 @@
<itunes:summary><%= item.summary %></itunes:summary> <itunes:summary><%= item.summary %></itunes:summary>
<itunes:keywords><![CDATA[<%= item.keywords |> Enum.join(", ") %>]]></itunes:keywords> <itunes:keywords><![CDATA[<%= item.keywords |> Enum.join(", ") %>]]></itunes:keywords>
<itunes:explicit><%= item.explicit %></itunes:explicit> <itunes:explicit><%= item.explicit %></itunes:explicit>
<%= if item.image != nil do %> <itunes:image href="<%= item.image || @podcast.image.url %>" />
<itunes:image href="<%= item.image %>" />
<% end %>
</item> </item>
<% end %> <% end %>
</channel> </channel>

45
test/podcast_feed/provider/archive/parser_test.exs
View file

@ -4,14 +4,21 @@ defmodule PodcastFeed.Provider.Archive.ParserTest do
alias PodcastFeed.Provider.Archive.Parser alias PodcastFeed.Provider.Archive.Parser
setup_all do setup_all do
valid_json = "{\"created\":1590247789,\"d1\":\"ia601402.us.archive.org\",\"d2\":\"ia801402.us.archive.org\",\"dir\":\"/8/items/incontri-a-piano-terra\",\"files\":[{\"name\": \"metadata.json\",\"source\": \"original\",\"mtime\": \"1590258296\",\"size\": \"315\",\"md5\": \"a0c0e219cf3f13e54f2a4b3efef8e5c8\",\"crc32\": \"2d181b5c\",\"sha1\": \"8244b579a759edddd01c905dd11f8565e83d0898\",\"format\": \"JSON\"},{\"name\": \"cover.jpg\",\"source\": \"original\",\"mtime\": \"1590258445\",\"size\": \"10650\",\"md5\": \"15687b23e11f0099abbfe64eb1685c31\",\"crc32\": \"fbb1516a\",\"sha1\": \"98fa929c7554241cfa92bea8eba69b39c5d47603\",\"format\": \"JPEG\",\"rotation\": \"0\"},{\"name\":\"Confini mobili sulle alpi.mp3\",\"source\":\"original\",\"mtime\":\"1590135989\",\"size\":\"46933494\",\"md5\":\"e832ee9381a4f8af2d9727e2f49126ae\",\"crc32\":\"d709dd90\",\"sha1\":\"89c820a2dfd63cfbbf7aeefd191c653756b33fe3\",\"format\":\"VBR MP3\",\"length\":\"3902.35\",\"height\":\"0\",\"width\":\"0\",\"title\":\"Confini mobili sulle alpi (italian limes)\",\"creator\":\"APE Milano\",\"album\":\"Incontri a Piano Terra\",\"track\":\"02\",\"artist\":\"APE Milano\",\"genre\":\"podcast\"},{\"name\":\"Confini mobili sulle alpi.png\",\"source\":\"derivative\",\"format\":\"PNG\",\"original\":\"Confini mobili sulle alpi.mp3\",\"mtime\":\"1590137809\",\"size\":\"34656\",\"md5\":\"63893f9b00402a107682b5317e808523\",\"crc32\":\"b59ff609\",\"sha1\":\"a396716431cd0acedd243030093d0b31d792cfb3\"},{\"name\":\"Confini mobili sulle alpi_spectrogram.png\",\"source\":\"derivative\",\"format\":\"Spectrogram\",\"original\":\"Confini mobili sulle alpi.mp3\",\"mtime\":\"1590137854\",\"size\":\"273188\",\"md5\":\"557337665c6d9f962b2e91d169f25e1b\",\"crc32\":\"08b4b57c\",\"sha1\":\"88e088f9c4954aa8f0849b7e0d69cee8d7d42327\"}],\"files_count\":31,\"item_last_updated\":1590160774,\"item_size\":244544362,\"metadata\":{\"identifier\":\"incontri-a-piano-terra\",\"mediatype\":\"audio\",\"collection\":\"opensource_audio\",\"creator\":\"APE Milano\",\"description\":\"Qualche registrazione delle attivit\\u00e0 sociali che promuoviamo al Piano Terra di Milano\",\"language\":\"ita\",\"licenseurl\":\"https://creativecommons.org/licenses/by-nc-nd/4.0/\",\"scanner\":\"Internet Archive HTML5 Uploader 1.6.4\",\"subject\":[\"ape milano\",\"podcast\",\"montagna\"],\"title\":\"Incontri a Piano Terra\",\"uploader\":\"milanoape@gmail.com\",\"publicdate\":\"2020-05-22 08:30:21\",\"addeddate\":\"2020-05-22 08:30:21\",\"curation\":\"[curator]validator@archive.org[/curator][date]20200522085526[/date][comment]checked for malware[/comment]\"},\"server\":\"ia601402.us.archive.org\",\"uniq\":122833277,\"workable_servers\":[\"ia601402.us.archive.org\",\"ia801402.us.archive.org\"]}" json_metadata = "{\"created\":1590247789,\"d1\":\"ia601402.us.archive.org\",\"d2\":\"ia801402.us.archive.org\",\"dir\":\"/8/items/incontri-a-piano-terra\",\"files\":[{\"name\": \"metadata.json\",\"source\": \"original\",\"mtime\": \"1590258296\",\"size\": \"315\",\"md5\": \"a0c0e219cf3f13e54f2a4b3efef8e5c8\",\"crc32\": \"2d181b5c\",\"sha1\": \"8244b579a759edddd01c905dd11f8565e83d0898\",\"format\": \"JSON\"},{\"name\": \"cover.jpg\",\"source\": \"original\",\"mtime\": \"1590258445\",\"size\": \"10650\",\"md5\": \"15687b23e11f0099abbfe64eb1685c31\",\"crc32\": \"fbb1516a\",\"sha1\": \"98fa929c7554241cfa92bea8eba69b39c5d47603\",\"format\": \"JPEG\",\"rotation\": \"0\"},{\"name\":\"Confini mobili sulle alpi.mp3\",\"source\":\"original\",\"mtime\":\"1590135989\",\"size\":\"46933494\",\"md5\":\"e832ee9381a4f8af2d9727e2f49126ae\",\"crc32\":\"d709dd90\",\"sha1\":\"89c820a2dfd63cfbbf7aeefd191c653756b33fe3\",\"format\":\"VBR MP3\",\"length\":\"3902.35\",\"height\":\"0\",\"width\":\"0\",\"title\":\"Confini mobili sulle alpi (italian limes)\",\"creator\":\"APE Milano\",\"album\":\"Incontri a Piano Terra\",\"track\":\"02\",\"artist\":\"APE Milano\",\"genre\":\"podcast\"},{\"name\":\"Confini mobili sulle alpi.png\",\"source\":\"derivative\",\"format\":\"PNG\",\"original\":\"Confini mobili sulle alpi.mp3\",\"mtime\":\"1590137809\",\"size\":\"34656\",\"md5\":\"63893f9b00402a107682b5317e808523\",\"crc32\":\"b59ff609\",\"sha1\":\"a396716431cd0acedd243030093d0b31d792cfb3\"},{\"name\":\"Confini mobili sulle alpi_spectrogram.png\",\"source\":\"derivative\",\"format\":\"Spectrogram\",\"original\":\"Confini mobili sulle alpi.mp3\",\"mtime\":\"1590137854\",\"size\":\"273188\",\"md5\":\"557337665c6d9f962b2e91d169f25e1b\",\"crc32\":\"08b4b57c\",\"sha1\":\"88e088f9c4954aa8f0849b7e0d69cee8d7d42327\"}],\"files_count\":31,\"item_last_updated\":1590160774,\"item_size\":244544362,\"metadata\":{\"identifier\":\"incontri-a-piano-terra\",\"mediatype\":\"audio\",\"collection\":\"opensource_audio\",\"creator\":\"APE Milano\",\"description\":\"Qualche registrazione delle attivit\\u00e0 sociali che promuoviamo al Piano Terra di Milano\",\"language\":\"ita\",\"licenseurl\":\"https://creativecommons.org/licenses/by-nc-nd/4.0/\",\"scanner\":\"Internet Archive HTML5 Uploader 1.6.4\",\"subject\":[\"ape milano\",\"podcast\",\"montagna\"],\"title\":\"Incontri a Piano Terra\",\"uploader\":\"milanoape@gmail.com\",\"publicdate\":\"2020-05-22 08:30:21\",\"addeddate\":\"2020-05-22 08:30:21\",\"curation\":\"[curator]validator@archive.org[/curator][date]20200522085526[/date][comment]checked for malware[/comment]\"},\"server\":\"ia601402.us.archive.org\",\"uniq\":122833277,\"workable_servers\":[\"ia601402.us.archive.org\",\"ia801402.us.archive.org\"]}"
extra = "{\"link\": \"http://www.ape-alveare.it/\",\"image\": {\"url\": \"http://www.ape-alveare.it/wp-content/themes/yootheme/cache/2018_logo_Ape_righe_trasparenza-d1aae6b9.png\",\"title\": \"APE Milano\",\"link\": \"http://www.ape-alveare.it/\"},\"category\": \"Montagna\",\"explicit\": \"no\" }" json_custom = "{\"link\": \"http://www.ape-alveare.it/\",\"image\": {\"url\": \"http://www.ape-alveare.it/wp-content/themes/yootheme/cache/2018_logo_Ape_righe_trasparenza-d1aae6b9.png\",\"title\": \"APE Milano\",\"link\": \"http://www.ape-alveare.it/\"},\"category\": \"Montagna\",\"explicit\": \"no\" }"
{:ok, json: valid_json, extra: extra} token = %Parser{
identifier: "incontri-a-piano-terra",
archive_metadata: Poison.decode!(json_metadata),
custom_metadata: Poison.decode!(json_custom),
}
{:ok, token: token}
end end
test "podcast data are correctly converted", state do test "podcast data are correctly converted", state do
%{podcast: podcast} = Parser.parse("incontri-a-piano-terra", Poison.decode!(state[:json]), Poison.decode!(state[:extra])) %{podcast: podcast} = Parser.to_podcast_feed_data(state[:token])
assert %{ assert %{
title: "Incontri a Piano Terra", title: "Incontri a Piano Terra",
description: "Qualche registrazione delle attività sociali che promuoviamo al Piano Terra di Milano", description: "Qualche registrazione delle attività sociali che promuoviamo al Piano Terra di Milano",
@ -37,8 +44,33 @@ defmodule PodcastFeed.Provider.Archive.ParserTest do
} == podcast } == podcast
end end
test "using first `original` image found on archive when image is missing on custom metadata", state do
custom = %{state[:token].custom_metadata | "image" => %{
"url" => nil,
"title" => nil,
"link" => nil,
}}
%{podcast: podcast} = Parser.to_podcast_feed_data(%Parser{state[:token] | custom_metadata: custom})
assert %{
image: %{
url: "https://archive.org/download/incontri-a-piano-terra/cover.jpg",
title: "Incontri a Piano Terra",
link: "https://archive.org/details/incontri-a-piano-terra",
},
} = podcast
end
test "using archive item details as link when link is missing on custom metadata", state do
custom = %{state[:token].custom_metadata | "link" => nil}
%{podcast: podcast} = Parser.to_podcast_feed_data(%Parser{state[:token] | custom_metadata: custom})
assert %{link: "https://archive.org/details/incontri-a-piano-terra"} = podcast
end
test "items data are correctly converted", state do test "items data are correctly converted", state do
%{items: items} = Parser.parse("incontri-a-piano-terra", Poison.decode!(state[:json]), Poison.decode!(state[:extra])) %{items: items} = Parser.to_podcast_feed_data(state[:token])
assert [ assert [
%{ %{
title: "Confini mobili sulle alpi (italian limes)", title: "Confini mobili sulle alpi (italian limes)",
@ -49,7 +81,8 @@ defmodule PodcastFeed.Provider.Archive.ParserTest do
summary: "", summary: "",
keywords: ["Incontri a Piano Terra", "APE Milano", "podcast"], keywords: ["Incontri a Piano Terra", "APE Milano", "podcast"],
explicit: "no", explicit: "no",
image: "https://archive.org/download/incontri-a-piano-terra/Confini%20mobili%20sulle%20alpi.png", # image: "https://archive.org/download/incontri-a-piano-terra/Confini%20mobili%20sulle%20alpi.png",
image: nil,
size: "46933494" size: "46933494"
} }
] == items ] == items