feat: fetching data from archive.org at compile time

fetching data from archive.org at compile time to populate the
"chi-siamo" page
This commit is contained in:
danilo silva 2020-11-14 22:52:41 +00:00
parent 20de037048
commit 054d875b66
8 changed files with 85 additions and 63 deletions

View file

@ -79,6 +79,12 @@ h6 {
font-size: 2rem;
}
.author {
font-size: 14px;
margin-top: -8px;
margin-bottom: 10px;
}
@media (min-width: 576px) {
}

View file

@ -19,7 +19,16 @@ config :openpod, OpenpodWeb.Endpoint,
live_view: [signing_salt: "6jHv46Xw"]
config :openpod,
cache_duration: :timer.hours(24)
cache_duration: :timer.hours(24),
base_url: "https://openpod.abbiamoundominio.org/",
podders: [
"uau-pod",
"fuori-fase",
"incontri-a-piano-terra",
"la_blogosfera_podcast",
"prnt_scrn_podcast",
"anarcotraffico-001-20200611.128"
]
# Configures Elixir's Logger
config :logger, :console,
@ -31,7 +40,7 @@ config :phoenix, :json_library, Jason
config :openpod, Openpod.Boundary.ArchiveScheduler,
jobs: [
{"0 6-18 * * *", {Openpod.Boundary.ArchiveReloader, :reload, []}},
{"0 6-18 * * *", {Openpod.Boundary.ArchiveReloader, :reload, []}}
]
# Import environment specific config. This must remain at the bottom

View file

@ -16,15 +16,20 @@ defmodule Openpod.Provider.Archive.Parser do
"image" => %{
"url" => nil,
"title" => nil,
"link" => nil,
"link" => nil
},
"category" => "",
"explicit" => "no",
"version" => "1",
"version" => "1"
}
@enforce_keys [:identifier]
defstruct [:identifier, :podcast_data, :archive_metadata, custom_metadata: @custom_metadata_defaults]
defstruct [
:identifier,
:podcast_data,
:archive_metadata,
custom_metadata: @custom_metadata_defaults
]
def by_identifier(identifier) do
%Parser{identifier: identifier}
@ -39,8 +44,13 @@ defmodule Openpod.Provider.Archive.Parser do
}
end
defp podcast_data(token = %{archive_metadata: %{"metadata" => metadata, "item_last_updated" => last_updated}}) do
defp podcast_data(
token = %{
archive_metadata: %{"metadata" => metadata, "item_last_updated" => last_updated}
}
) do
link = Format.compile(@podcast_link, identifier: token.identifier)
%{
title: metadata["title"],
description: metadata["description"],
@ -48,21 +58,22 @@ defmodule Openpod.Provider.Archive.Parser do
managingEditor: metadata["uploader"],
owner: %{
name: metadata["creator"],
email: metadata["uploader"],
email: metadata["uploader"]
},
keywords: parse_subject(metadata["subject"]),
pubDate: metadata["publicdate"] |> NaiveDateTime.from_iso8601!() |> DateTime.from_naive!("Etc/UTC"),
pubDate:
metadata["publicdate"] |> NaiveDateTime.from_iso8601!() |> DateTime.from_naive!("Etc/UTC"),
lastBuildDate: last_updated |> DateTime.from_unix!(:second),
author: metadata["creator"],
language: ISO639.to_iso639_1(metadata["language"]),
image: %{
url: fetch_cover(token),
title: metadata["title"],
link: Map.get(metadata, "op_link") || link,
link: Map.get(metadata, "op_link") || link
},
link: Map.get(metadata, "op_link") || link,
category: Map.get(metadata, "op_category", ""),
explicit: Map.get(metadata, "op_explicit", "no"),
explicit: Map.get(metadata, "op_explicit", "no")
}
end
@ -74,41 +85,52 @@ defmodule Openpod.Provider.Archive.Parser do
defp fetch_archive_metadata(identifier) do
metadata_url = Format.compile(@archive_metadata_url, identifier: identifier)
{:ok, 200, _headers, client_ref} = :hackney.get(metadata_url, [], "", [follow_redirect: true, connect_timeout: 30_000, recv_timeout: 30_000])
{:ok, 200, _headers, client_ref} =
:hackney.get(metadata_url, [], "",
follow_redirect: true,
connect_timeout: 30_000,
recv_timeout: 30_000
)
{:ok, metadata_json} = :hackney.body(client_ref)
metadata_json |> Jason.decode!()
end
defp filter_audio_files(files) do
files |> Enum.filter(fn f -> Map.get(f, "format") =~ ~r/MP3/i end) #FIXME:! mp3, ogg, boh
# FIXME:! mp3, ogg, boh
files |> Enum.filter(fn f -> Map.get(f, "format") =~ ~r/MP3/i end)
end
defp to_feed_item(file, identifier, _files) do
filename = Map.get(file, "name")
%{
title: file["title"],
description: "",
pubDate: file |> Map.get("mtime") |> Integer.parse() |> elem(0) |> DateTime.from_unix!(:second),
pubDate:
file |> Map.get("mtime") |> Integer.parse() |> elem(0) |> DateTime.from_unix!(:second),
link: download_url(identifier, filename),
length: (file |> Map.get("length") |> Float.parse() |> elem(0)) |> trunc(),
length: file |> Map.get("length") |> Float.parse() |> elem(0) |> trunc(),
size: file |> Map.get("size"),
summary: "",
# image: download_url(identifier, fetch_image_of_audio(filename, files)),
image: nil,
keywords: file |> Map.take(["album", "artist", "genre"]) |> Map.values(),
explicit: "no",
explicit: "no"
}
end
defp fetch_cover(%{identifier: identifier, archive_metadata: %{"files" => files}}) do
filename = files
|> Enum.filter(fn f -> f["source"] == "original" end)
|> Enum.filter(fn f -> f["format"] =~ ~r/JPG|JPEG|PNG|GIF|Item Image/i end)
|> List.first()
|> case do
nil -> nil
file -> Map.get(file, "name")
end
filename =
files
|> Enum.filter(fn f -> f["source"] == "original" end)
|> Enum.filter(fn f -> f["format"] =~ ~r/JPG|JPEG|PNG|GIF|Item Image/i end)
|> List.first()
|> case do
nil -> nil
file -> Map.get(file, "name")
end
download_url(identifier, filename)
end
@ -128,6 +150,7 @@ defmodule Openpod.Provider.Archive.Parser do
# defp fetch_image_of_audio(image_file), do: image_file |> Map.get("name", nil)
defp download_url(_identifier, nil), do: nil
defp download_url(identifier, filename) do
Format.compile(@download_url, identifier: identifier, filename: filename) |> URI.encode()
end

View file

@ -62,6 +62,7 @@ defmodule OpenpodWeb do
quote do
# Use all HTML functionality (forms, tags, etc)
use Phoenix.HTML
use PhoenixHtmlSanitizer, :basic_html
# Import basic rendering functionality (render, render_layout, etc)
import Phoenix.View

View file

@ -1,6 +1,17 @@
defmodule OpenpodWeb.PageController do
use OpenpodWeb, :controller
Application.ensure_all_started(:hackney)
@base_url Application.get_env(:openpod, :base_url)
@podders Application.get_env(:openpod, :podders)
|> Enum.map(fn podcast ->
Openpod.Provider.Archive.Parser.by_identifier(podcast)
|> Map.get(:podcast)
|> Map.put(:podcast_url, "#{@base_url}/podcast/#{podcast}")
end)
def index(conn, _params) do
render(conn, "index.html")
end
@ -10,39 +21,6 @@ defmodule OpenpodWeb.PageController do
end
def chi(conn, _params) do
render(conn, "chi.html",
podders: [
%{
cover: "https://ia801504.us.archive.org/0/items/uau-pod/logo.jpg?cnt=0",
podcast_url: "https://openpod.abbiamoundominio.org/podcast/uau-pod",
name: "UAU POD",
description: """
Il podcast di Un'ambigua utopia anticipa l'uscita del numero speciale, il numero 10, dell'omonima rivista.
Archivio digitale ed altri materiali all'indirizzo <a href="http://archivio-uau.online/">http://archivio-uau.online/</a>
"""
},
%{
cover: "https://ia803204.us.archive.org/24/items/fuori-fase/fuori_fase_cover.jpg?cnt=0",
podcast_url: "https://openpod.abbiamoundominio.org/podcast/fuori-fase",
name: "Fuori Fase",
description: """
Come è stata gestita lemergenza nei suoi diversi aspetti? Quali sono state le conseguenze della pandemia, a livello economico e sociale?
Con oggi lanciamo il primo di una serie di contributi audio, frutto degli incontri e degli scambi realizzati lungo la quarantena e i lunghi mesi di lockdown,
per riflettere sui cambiamenti che lepidemia intesa come fenomeno sociale ha imposto su una serie di aspetti strutturali del nostro tempo.
Fuori Fase lo abbiamo chiamato: appunti per riscrivere il crono-programma dellemergenza e del dopo-Covid da qui ai prossimi mesi.
<a href="https://offtopiclab.org/">https://offtopiclab.org/</a>
"""
},
%{
cover:
"https://ia801905.us.archive.org/15/items/incontri-a-piano-terra/tracce.jpg?cnt=0",
podcast_url: "https://openpod.abbiamoundominio.org/podcast/incontri-a-piano-terra",
name: "APE podcast",
description: """
Qualche registrazione delle attività sociali che promuoviamo al Piano Terra di Milano. <a href="http://ape-alveare.it/">http://ape-alveare.it</a>
"""
}
]
)
render(conn, "chi.html", podders: @podders)
end
end

View file

@ -22,16 +22,17 @@
<div class="col-12 col-md-2 border-right-red d-flex flex-column align-items-center justify-content-start">
</div>
<div class="col-12 col-md-3 pt-3 pb-3">
<img class="img-fluid" src="<%= podder.cover %>">
<img class="img-fluid" src="<%= podder.image.url %>">
</div>
<div class="col-12 col-md-4 pt-3">
<div class="d-flex align-items-center mb-2">
<h4 class="m-0 mr-2 mt-1"><%= podder.name %></h4>
<h4 class="m-0 mr-2 mt-1"><%= podder.title %></h4>
<a role="button" href="<%= podder.podcast_url %>" target="_blank" class="btn btn-sm btn-danger"><svg width="1em" height="1em" viewBox="0 0 16 16" class="bi bi-rss-fill" fill="currentColor" xmlns="http://www.w3.org/2000/svg">
<path fill-rule="evenodd" d="M2 0a2 2 0 0 0-2 2v12a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V2a2 2 0 0 0-2-2H2zm1.5 2.5a1 1 0 0 0 0 2 8 8 0 0 1 8 8 1 1 0 1 0 2 0c0-5.523-4.477-10-10-10zm0 4a1 1 0 0 0 0 2 4 4 0 0 1 4 4 1 1 0 1 0 2 0 6 6 0 0 0-6-6zm.5 7a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3z"/>
</svg></a>
</div>
<p><%= raw podder.description %></p>
<div class="author"><a href="<%= podder.link %>" target="_blank"><%= podder.author %></a></div>
<p><%= sanitize podder.description, :basic_html %></p>
</div>
</div>
<% end %>

View file

@ -12,7 +12,7 @@ defmodule Openpod.MixProject do
preferred_cli_env: [coveralls: :test, e2e: :test, "coveralls.html": :test],
start_permanent: Mix.env() == :prod,
aliases: aliases(),
deps: deps(),
deps: deps()
]
end
@ -51,6 +51,7 @@ defmodule Openpod.MixProject do
{:quantum, "~> 3.0-rc"},
{:observer_cli, "~> 1.5"},
{:iso639_elixir, "~> 0.1.0"},
{:phoenix_html_sanitizer, "~> 1.0.0"}
] ++ deps_dev() ++ deps_release()
end
@ -58,7 +59,7 @@ defmodule Openpod.MixProject do
[
{:credo, "~> 1.4", only: [:dev, :test], runtime: false},
{:dialyxir, "~> 1.0", only: [:dev], runtime: false},
{:excoveralls, "~> 0.7", only: [:dev, :test]},
{:excoveralls, "~> 0.7", only: [:dev, :test]}
]
end
@ -76,7 +77,7 @@ defmodule Openpod.MixProject do
# See the documentation for `Mix` for more info on aliases.
defp aliases do
[
setup: ["deps.get", "cmd npm install --prefix assets"],
setup: ["deps.get", "cmd npm install --prefix assets"]
]
end
end

View file

@ -18,16 +18,19 @@
"gettext": {:hex, :gettext, "0.18.1", "89e8499b051c7671fa60782faf24409b5d2306aa71feb43d79648a8bc63d0522", [:mix], [], "hexpm", "e70750c10a5f88cb8dc026fc28fa101529835026dec4a06dba3b614f2a99c7a9"},
"hackney": {:hex, :hackney, "1.16.0", "5096ac8e823e3a441477b2d187e30dd3fff1a82991a806b2003845ce72ce2d84", [:rebar3], [{:certifi, "2.5.2", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.1", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.3.0", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.6", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm", "3bf0bebbd5d3092a3543b783bf065165fa5d3ad4b899b836810e513064134e18"},
"html_entities": {:hex, :html_entities, "0.5.1", "1c9715058b42c35a2ab65edc5b36d0ea66dd083767bef6e3edb57870ef556549", [:mix], [], "hexpm", "30efab070904eb897ff05cd52fa61c1025d7f8ef3a9ca250bc4e6513d16c32de"},
"html_sanitize_ex": {:hex, :html_sanitize_ex, "1.0.1", "2572e7122c78ab7e57b613e7c7f5e42bf9b3c25e430e32f23f1413d86db8a0af", [:mix], [{:mochiweb, "~> 2.12.2", [hex: :mochiweb, repo: "hexpm", optional: false]}], "hexpm", "c334e2835e094fb9c04658bd4cfc7533fa51a8f56f11343c57ab9cb2a01d8613"},
"idna": {:hex, :idna, "6.0.1", "1d038fb2e7668ce41fbf681d2c45902e52b3cb9e9c77b55334353b222c2ee50c", [:rebar3], [{:unicode_util_compat, "0.5.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "a02c8a1c4fd601215bb0b0324c8a6986749f807ce35f25449ec9e69758708122"},
"iso639_elixir": {:hex, :iso639_elixir, "0.1.0", "805b689ebc1144f25424ad4f97d6991135680ec37ba2e170583696e73bf1574e", [:mix], [{:poison, "~> 3.0", [hex: :poison, repo: "hexpm", optional: false]}], "hexpm", "1cb9f423e93b8785783ab9d694de47377577f77a0ce00ac6b9a74730368a726b"},
"jason": {:hex, :jason, "1.2.2", "ba43e3f2709fd1aa1dce90aaabfd039d000469c05c56f0b8e31978e03fa39052", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "18a228f5f0058ee183f29f9eae0805c6e59d61c3b006760668d8d18ff0d12179"},
"metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"},
"mime": {:hex, :mime, "1.4.0", "5066f14944b470286146047d2f73518cf5cca82f8e4815cf35d196b58cf07c47", [:mix], [], "hexpm", "75fa42c4228ea9a23f70f123c74ba7cece6a03b1fd474fe13f6a7a85c6ea4ff6"},
"mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm", "f278585650aa581986264638ebf698f8bb19df297f66ad91b18910dfc6e19323"},
"mochiweb": {:hex, :mochiweb, "2.12.2", "80804ad342afa3d7f3524040d4eed66ce74b17a555de454ac85b07c479928e46", [:make, :rebar], [], "hexpm", "d3e681d4054b74a96cf2efcd09e94157ab83a5f55ddc4ce69f90b8144673bd7a"},
"observer_cli": {:hex, :observer_cli, "1.5.4", "e8489b3a7c77c2155c0b67fa9f90d9afa76bf15c24fb7b312addcc117771d154", [:mix, :rebar3], [{:recon, "~>2.5.1", [hex: :recon, repo: "hexpm", optional: false]}], "hexpm", "f4825d374b22f258ad114a74796b79b86e20a03611051719f6062353c05389cb"},
"parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm", "17ef63abde837ad30680ea7f857dd9e7ced9476cdd7b0394432af4bfc241b960"},
"phoenix": {:hex, :phoenix, "1.5.4", "0fca9ce7e960f9498d6315e41fcd0c80bfa6fbeb5fa3255b830c67fdfb7e703f", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_html, "~> 2.13", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.0", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:plug, "~> 1.10", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 1.0 or ~> 2.2", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.1.2 or ~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "4e516d131fde87b568abd62e1b14aa07ba7d5edfd230bab4e25cc9dedbb39135"},
"phoenix_html": {:hex, :phoenix_html, "2.14.2", "b8a3899a72050f3f48a36430da507dd99caf0ac2d06c77529b1646964f3d563e", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "58061c8dfd25da5df1ea0ca47c972f161beb6c875cd293917045b92ffe1bf617"},
"phoenix_html_sanitizer": {:hex, :phoenix_html_sanitizer, "1.0.2", "e2c8cfbc83660e362753de127cc957bec3442a8aecdf271fb65a684a906fccf5", [:mix], [{:html_sanitize_ex, "~> 1.0.0", [hex: :html_sanitize_ex, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.0", [hex: :phoenix_html, repo: "hexpm", optional: false]}], "hexpm", "47aebb08fa954b7ad95f295fb701df9800ee3a489212119c9c6074a65e1e5a10"},
"phoenix_live_dashboard": {:hex, :phoenix_live_dashboard, "0.2.7", "21564144897109ac486518651fecd09403a4d9df4d8432e7dcdf156df6a6a31a", [:mix], [{:phoenix_html, "~> 2.14.1 or ~> 2.15", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:phoenix_live_view, "~> 0.14.0", [hex: :phoenix_live_view, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.4.0 or ~> 0.5.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "2204c2c6755da7b39a21e312253b93d977cc846c85df8a6c0d9f9505cd8bf15b"},
"phoenix_live_reload": {:hex, :phoenix_live_reload, "1.2.4", "940c0344b1d66a2e46eef02af3a70e0c5bb45a4db0bf47917add271b76cd3914", [:mix], [{:file_system, "~> 0.2.1 or ~> 0.3", [hex: :file_system, repo: "hexpm", optional: false]}, {:phoenix, "~> 1.4", [hex: :phoenix, repo: "hexpm", optional: false]}], "hexpm", "38f9308357dea4cc77f247e216da99fcb0224e05ada1469167520bed4cb8cccd"},
"phoenix_live_view": {:hex, :phoenix_live_view, "0.14.4", "7286a96287cd29b594ce4a7314249cea7311af04a06c0fa3e50932e188e73996", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix, "~> 1.5.3", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.14", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.2 or ~> 0.5", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "fc4f8cf205c784eeccee35de8afbfeb995ce5511ac4839db63d6d67a5ba091d1"},