diff --git a/lib/podcast_feed/provider/archive/parser.ex b/lib/podcast_feed/provider/archive/parser.ex index 6041ad9..9019949 100644 --- a/lib/podcast_feed/provider/archive/parser.ex +++ b/lib/podcast_feed/provider/archive/parser.ex @@ -7,7 +7,6 @@ defmodule PodcastFeed.Provider.Archive.Parser do alias PodcastFeed.Utility.Format alias __MODULE__ - @custom_metadata_url "https://archive.org/download/{identifier}/metadata.json" @archive_metadata_url "http://archive.org/metadata/{identifier}" @download_url "https://archive.org/download/{identifier}/{filename}" @podcast_link "https://archive.org/details/{identifier}" @@ -30,7 +29,6 @@ defmodule PodcastFeed.Provider.Archive.Parser do def by_identifier(identifier) do %Parser{identifier: identifier} |> enrich_with_archive_metadata() - |> enrich_with_custom_metadata() |> to_podcast_feed_data() end @@ -41,7 +39,7 @@ defmodule PodcastFeed.Provider.Archive.Parser do } end - defp podcast_data(token = %{custom_metadata: custom, archive_metadata: %{"metadata" => metadata, "item_last_updated" => last_updated}}) do + defp podcast_data(token = %{archive_metadata: %{"metadata" => metadata, "item_last_updated" => last_updated}}) do link = Format.compile(@podcast_link, identifier: token.identifier) %{ title: metadata["title"], @@ -58,13 +56,13 @@ defmodule PodcastFeed.Provider.Archive.Parser do author: metadata["creator"], language: metadata["language"], image: %{ - url: get_in(custom, ["image", "url"]) || fetch_cover(token), - title: get_in(custom, ["image", "title"]) || metadata["title"], - link: get_in(custom, ["image", "link"]) || link, + url: fetch_cover(token), + title: metadata["title"], + link: Map.get(metadata, "op_link") || link, }, - link: Map.get(custom, "link") || link, - category: Map.get(custom, "category", ""), - explicit: Map.get(custom, "explicit", "no"), + link: Map.get(metadata, "op_link") || link, + category: Map.get(metadata, "op_category", ""), + explicit: Map.get(metadata, "op_explicit", "no"), } end @@ -74,20 +72,6 @@ defmodule PodcastFeed.Provider.Archive.Parser do |> Enum.map(fn f -> to_feed_item(f, identifier, files) end) end - defp fetch_custom_metadata(identifier) do - custom_metadata_url = Format.compile(@custom_metadata_url, identifier: identifier) - parse_custom_metadata_response(:hackney.get(custom_metadata_url, [], "", [follow_redirect: true])) - end - - defp parse_custom_metadata_response({:ok, 200, _headers, client_ref}) do - {:ok, custom_metadata_json} = :hackney.body(client_ref) - custom_metadata_json - |> String.split("\n") - |> Enum.join() - |> Jason.decode!() - end - defp parse_custom_metadata_response(_), do: @custom_metadata_defaults - defp fetch_archive_metadata(identifier) do metadata_url = Format.compile(@archive_metadata_url, identifier: identifier) {:ok, 200, _headers, client_ref} = :hackney.get(metadata_url, [], "", [follow_redirect: true, connect_timeout: 30_000, recv_timeout: 30_000]) @@ -152,10 +136,6 @@ defmodule PodcastFeed.Provider.Archive.Parser do %Parser{token | archive_metadata: fetch_archive_metadata(token.identifier)} end - defp enrich_with_custom_metadata(token) do - %Parser{token | custom_metadata: fetch_custom_metadata(token.identifier)} - end - defp parse_subject(subject) when is_list(subject), do: subject defp parse_subject(subject) when is_binary(subject), do: subject |> String.split(";") end diff --git a/test/podcast_feed/provider/archive/parser_test.exs b/test/podcast_feed/provider/archive/parser_test.exs index cd70c88..2ccaeab 100644 --- a/test/podcast_feed/provider/archive/parser_test.exs +++ b/test/podcast_feed/provider/archive/parser_test.exs @@ -4,13 +4,11 @@ defmodule PodcastFeed.Provider.Archive.ParserTest do alias PodcastFeed.Provider.Archive.Parser setup_all do - json_metadata = "{\"created\":1590247789,\"d1\":\"ia601402.us.archive.org\",\"d2\":\"ia801402.us.archive.org\",\"dir\":\"/8/items/incontri-a-piano-terra\",\"files\":[{\"name\": \"metadata.json\",\"source\": \"original\",\"mtime\": \"1590258296\",\"size\": \"315\",\"md5\": \"a0c0e219cf3f13e54f2a4b3efef8e5c8\",\"crc32\": \"2d181b5c\",\"sha1\": \"8244b579a759edddd01c905dd11f8565e83d0898\",\"format\": \"JSON\"},{\"name\": \"cover.jpg\",\"source\": \"original\",\"mtime\": \"1590258445\",\"size\": \"10650\",\"md5\": \"15687b23e11f0099abbfe64eb1685c31\",\"crc32\": \"fbb1516a\",\"sha1\": \"98fa929c7554241cfa92bea8eba69b39c5d47603\",\"format\": \"JPEG\",\"rotation\": \"0\"},{\"name\":\"Confini mobili sulle alpi.mp3\",\"source\":\"original\",\"mtime\":\"1590135989\",\"size\":\"46933494\",\"md5\":\"e832ee9381a4f8af2d9727e2f49126ae\",\"crc32\":\"d709dd90\",\"sha1\":\"89c820a2dfd63cfbbf7aeefd191c653756b33fe3\",\"format\":\"VBR MP3\",\"length\":\"3902.35\",\"height\":\"0\",\"width\":\"0\",\"title\":\"Confini mobili sulle alpi (italian limes)\",\"creator\":\"APE Milano\",\"album\":\"Incontri a Piano Terra\",\"track\":\"02\",\"artist\":\"APE Milano\",\"genre\":\"podcast\"},{\"name\":\"Confini mobili sulle alpi.png\",\"source\":\"derivative\",\"format\":\"PNG\",\"original\":\"Confini mobili sulle alpi.mp3\",\"mtime\":\"1590137809\",\"size\":\"34656\",\"md5\":\"63893f9b00402a107682b5317e808523\",\"crc32\":\"b59ff609\",\"sha1\":\"a396716431cd0acedd243030093d0b31d792cfb3\"},{\"name\":\"Confini mobili sulle alpi_spectrogram.png\",\"source\":\"derivative\",\"format\":\"Spectrogram\",\"original\":\"Confini mobili sulle alpi.mp3\",\"mtime\":\"1590137854\",\"size\":\"273188\",\"md5\":\"557337665c6d9f962b2e91d169f25e1b\",\"crc32\":\"08b4b57c\",\"sha1\":\"88e088f9c4954aa8f0849b7e0d69cee8d7d42327\"}],\"files_count\":31,\"item_last_updated\":1590160774,\"item_size\":244544362,\"metadata\":{\"identifier\":\"incontri-a-piano-terra\",\"mediatype\":\"audio\",\"collection\":\"opensource_audio\",\"creator\":\"APE Milano\",\"description\":\"Qualche registrazione delle attivit\\u00e0 sociali che promuoviamo al Piano Terra di Milano\",\"language\":\"ita\",\"licenseurl\":\"https://creativecommons.org/licenses/by-nc-nd/4.0/\",\"scanner\":\"Internet Archive HTML5 Uploader 1.6.4\",\"subject\":[\"ape milano\",\"podcast\",\"montagna\"],\"title\":\"Incontri a Piano Terra\",\"uploader\":\"milanoape@gmail.com\",\"publicdate\":\"2020-05-22 08:30:21\",\"addeddate\":\"2020-05-22 08:30:21\",\"curation\":\"[curator]validator@archive.org[/curator][date]20200522085526[/date][comment]checked for malware[/comment]\"},\"server\":\"ia601402.us.archive.org\",\"uniq\":122833277,\"workable_servers\":[\"ia601402.us.archive.org\",\"ia801402.us.archive.org\"]}" - json_custom = "{\"link\": \"http://www.ape-alveare.it/\",\"image\": {\"url\": \"http://www.ape-alveare.it/wp-content/themes/yootheme/cache/2018_logo_Ape_righe_trasparenza-d1aae6b9.png\",\"title\": \"APE Milano\",\"link\": \"http://www.ape-alveare.it/\"},\"category\": \"Montagna\",\"explicit\": \"no\" }" + json_metadata = "{\"created\":1590247789,\"d1\":\"ia601402.us.archive.org\",\"d2\":\"ia801402.us.archive.org\",\"dir\":\"/8/items/incontri-a-piano-terra\",\"files\":[{\"name\": \"metadata.json\",\"source\": \"original\",\"mtime\": \"1590258296\",\"size\": \"315\",\"md5\": \"a0c0e219cf3f13e54f2a4b3efef8e5c8\",\"crc32\": \"2d181b5c\",\"sha1\": \"8244b579a759edddd01c905dd11f8565e83d0898\",\"format\": \"JSON\"},{\"name\": \"cover.jpg\",\"source\": \"original\",\"mtime\": \"1590258445\",\"size\": \"10650\",\"md5\": \"15687b23e11f0099abbfe64eb1685c31\",\"crc32\": \"fbb1516a\",\"sha1\": \"98fa929c7554241cfa92bea8eba69b39c5d47603\",\"format\": \"JPEG\",\"rotation\": \"0\"},{\"name\":\"Confini mobili sulle alpi.mp3\",\"source\":\"original\",\"mtime\":\"1590135989\",\"size\":\"46933494\",\"md5\":\"e832ee9381a4f8af2d9727e2f49126ae\",\"crc32\":\"d709dd90\",\"sha1\":\"89c820a2dfd63cfbbf7aeefd191c653756b33fe3\",\"format\":\"VBR MP3\",\"length\":\"3902.35\",\"height\":\"0\",\"width\":\"0\",\"title\":\"Confini mobili sulle alpi (italian limes)\",\"creator\":\"APE Milano\",\"album\":\"Incontri a Piano Terra\",\"track\":\"02\",\"artist\":\"APE Milano\",\"genre\":\"podcast\"},{\"name\":\"Confini mobili sulle alpi.png\",\"source\":\"derivative\",\"format\":\"PNG\",\"original\":\"Confini mobili sulle alpi.mp3\",\"mtime\":\"1590137809\",\"size\":\"34656\",\"md5\":\"63893f9b00402a107682b5317e808523\",\"crc32\":\"b59ff609\",\"sha1\":\"a396716431cd0acedd243030093d0b31d792cfb3\"},{\"name\":\"Confini mobili sulle alpi_spectrogram.png\",\"source\":\"derivative\",\"format\":\"Spectrogram\",\"original\":\"Confini mobili sulle alpi.mp3\",\"mtime\":\"1590137854\",\"size\":\"273188\",\"md5\":\"557337665c6d9f962b2e91d169f25e1b\",\"crc32\":\"08b4b57c\",\"sha1\":\"88e088f9c4954aa8f0849b7e0d69cee8d7d42327\"}],\"files_count\":31,\"item_last_updated\":1590160774,\"item_size\":244544362,\"metadata\":{\"identifier\":\"incontri-a-piano-terra\",\"mediatype\":\"audio\",\"collection\":\"opensource_audio\",\"creator\":\"APE Milano\",\"description\":\"Qualche registrazione delle attivit\\u00e0 sociali che promuoviamo al Piano Terra di Milano\",\"language\":\"ita\",\"licenseurl\":\"https://creativecommons.org/licenses/by-nc-nd/4.0/\",\"scanner\":\"Internet Archive HTML5 Uploader 1.6.4\",\"subject\":[\"ape milano\",\"podcast\",\"montagna\"],\"title\":\"Incontri a Piano Terra\",\"uploader\":\"milanoape@gmail.com\",\"publicdate\":\"2020-05-22 08:30:21\",\"addeddate\":\"2020-05-22 08:30:21\",\"curation\":\"[curator]validator@archive.org[/curator][date]20200522085526[/date][comment]checked for malware[/comment]\",\"op_link\":\"http://www.ape-alveare.it/\",\"op_category\":\"Montagna\",\"op_explicit\":\"no\"},\"server\":\"ia601402.us.archive.org\",\"uniq\":122833277,\"workable_servers\":[\"ia601402.us.archive.org\",\"ia801402.us.archive.org\"]}" token = %Parser{ identifier: "incontri-a-piano-terra", archive_metadata: Jason.decode!(json_metadata), - custom_metadata: Jason.decode!(json_custom), } {:ok, token: token} @@ -34,9 +32,9 @@ defmodule PodcastFeed.Provider.Archive.ParserTest do author: "APE Milano", language: "ita", image: %{ - url: "http://www.ape-alveare.it/wp-content/themes/yootheme/cache/2018_logo_Ape_righe_trasparenza-d1aae6b9.png", - title: "APE Milano", link: "http://www.ape-alveare.it/", + title: "Incontri a Piano Terra", + url: "https://archive.org/download/incontri-a-piano-terra/cover.jpg" }, link: "http://www.ape-alveare.it/", category: "Montagna", @@ -53,30 +51,6 @@ defmodule PodcastFeed.Provider.Archive.ParserTest do } = podcast end - test "using first `original` image found on archive when image is missing on custom metadata", state do - custom = %{state[:token].custom_metadata | "image" => %{ - "url" => nil, - "title" => nil, - "link" => nil, - }} - %{podcast: podcast} = Parser.to_podcast_feed_data(%Parser{state[:token] | custom_metadata: custom}) - - assert %{ - image: %{ - url: "https://archive.org/download/incontri-a-piano-terra/cover.jpg", - title: "Incontri a Piano Terra", - link: "https://archive.org/details/incontri-a-piano-terra", - }, - } = podcast - end - - test "using archive item details as link when link is missing on custom metadata", state do - custom = %{state[:token].custom_metadata | "link" => nil} - %{podcast: podcast} = Parser.to_podcast_feed_data(%Parser{state[:token] | custom_metadata: custom}) - - assert %{link: "https://archive.org/details/incontri-a-piano-terra"} = podcast - end - test "items data are correctly converted", state do %{items: items} = Parser.to_podcast_feed_data(state[:token])