FFmpeg
ipfsgateway.c
Go to the documentation of this file.
1 /*
2  * IPFS and IPNS protocol support through IPFS Gateway.
3  * Copyright (c) 2022 Mark Gaiser
4  *
5  * This file is part of FFmpeg.
6  *
7  * FFmpeg is free software; you can redistribute it and/or
8  * modify it under the terms of the GNU Lesser General Public
9  * License as published by the Free Software Foundation; either
10  * version 2.1 of the License, or (at your option) any later version.
11  *
12  * FFmpeg is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15  * Lesser General Public License for more details.
16  *
17  * You should have received a copy of the GNU Lesser General Public
18  * License along with FFmpeg; if not, write to the Free Software
19  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20  */
21 
22 #include "libavutil/avstring.h"
23 #include "libavutil/getenv_utf8.h"
24 #include "libavutil/opt.h"
25 #include <sys/stat.h>
26 #include "os_support.h"
27 #include "url.h"
28 
29 // Define the posix PATH_MAX if not there already.
30 // This fixes a compile issue for MSVC.
31 #ifndef PATH_MAX
32 #define PATH_MAX 4096
33 #endif
34 
35 typedef struct IPFSGatewayContext {
36  AVClass *class;
38  // Is filled by the -gateway argument and not changed after.
39  char *gateway;
40  // If the above gateway is non null, it will be copied into this buffer.
41  // Else this buffer will contain the auto detected gateway.
42  // In either case, the gateway to use will be in this buffer.
45 
46 // A best-effort way to find the IPFS gateway.
47 // Only the most appropiate gateway is set. It's not actually requested
48 // (http call) to prevent a potential slowdown in startup. A potential timeout
49 // is handled by the HTTP protocol.
51 {
52  IPFSGatewayContext *c = h->priv_data;
53  char ipfs_full_data_folder[PATH_MAX];
54  char ipfs_gateway_file[PATH_MAX];
55  struct stat st;
56  int stat_ret = 0;
57  int ret = AVERROR(EINVAL);
58  FILE *gateway_file = NULL;
59  char *env_ipfs_gateway, *env_ipfs_path;
60 
61  // Test $IPFS_GATEWAY.
62  env_ipfs_gateway = getenv_utf8("IPFS_GATEWAY");
63  if (env_ipfs_gateway != NULL) {
64  int printed = snprintf(c->gateway_buffer, sizeof(c->gateway_buffer),
65  "%s", env_ipfs_gateway);
66  freeenv_utf8(env_ipfs_gateway);
67  if (printed >= sizeof(c->gateway_buffer)) {
69  "The IPFS_GATEWAY environment variable "
70  "exceeds the maximum length. "
71  "We allow a max of %zu characters\n",
72  sizeof(c->gateway_buffer));
73  ret = AVERROR(EINVAL);
74  goto err;
75  }
76 
77  ret = 1;
78  goto err;
79  } else
80  av_log(h, AV_LOG_DEBUG, "$IPFS_GATEWAY is empty.\n");
81 
82  // We need to know the IPFS folder to - eventually - read the contents of
83  // the "gateway" file which would tell us the gateway to use.
84  env_ipfs_path = getenv_utf8("IPFS_PATH");
85  if (env_ipfs_path == NULL) {
86  int printed;
87  char *env_home = getenv_utf8("HOME");
88 
89  av_log(h, AV_LOG_DEBUG, "$IPFS_PATH is empty.\n");
90 
91  // Try via the home folder.
92  if (env_home == NULL) {
93  av_log(h, AV_LOG_WARNING, "$HOME appears to be empty.\n");
94  ret = AVERROR(EINVAL);
95  goto err;
96  }
97 
98  // Verify the composed path fits.
99  printed = snprintf(
100  ipfs_full_data_folder, sizeof(ipfs_full_data_folder),
101  "%s/.ipfs/", env_home);
102  freeenv_utf8(env_home);
103  if (printed >= sizeof(ipfs_full_data_folder)) {
105  "The IPFS data path exceeds the "
106  "max path length (%zu)\n",
107  sizeof(ipfs_full_data_folder));
108  ret = AVERROR(EINVAL);
109  goto err;
110  }
111 
112  // Stat the folder.
113  // It should exist in a default IPFS setup when run as local user.
114  stat_ret = stat(ipfs_full_data_folder, &st);
115 
116  if (stat_ret < 0) {
118  "Unable to find IPFS folder. We tried:\n"
119  "- $IPFS_PATH, which was empty.\n"
120  "- $HOME/.ipfs (full uri: %s) which doesn't exist.\n",
121  ipfs_full_data_folder);
122  ret = AVERROR(ENOENT);
123  goto err;
124  }
125  } else {
126  int printed = snprintf(
127  ipfs_full_data_folder, sizeof(ipfs_full_data_folder),
128  "%s", env_ipfs_path);
129  freeenv_utf8(env_ipfs_path);
130  if (printed >= sizeof(ipfs_full_data_folder)) {
132  "The IPFS_PATH environment variable "
133  "exceeds the maximum length. "
134  "We allow a max of %zu characters\n",
135  sizeof(c->gateway_buffer));
136  ret = AVERROR(EINVAL);
137  goto err;
138  }
139  }
140 
141  // Copy the fully composed gateway path into ipfs_gateway_file.
142  if (snprintf(ipfs_gateway_file, sizeof(ipfs_gateway_file), "%sgateway",
143  ipfs_full_data_folder)
144  >= sizeof(ipfs_gateway_file)) {
146  "The IPFS gateway file path exceeds "
147  "the max path length (%zu)\n",
148  sizeof(ipfs_gateway_file));
149  ret = AVERROR(ENOENT);
150  goto err;
151  }
152 
153  // Get the contents of the gateway file.
154  gateway_file = avpriv_fopen_utf8(ipfs_gateway_file, "r");
155  if (!gateway_file) {
157  "The IPFS gateway file (full uri: %s) doesn't exist. "
158  "Is the gateway enabled?\n",
159  ipfs_gateway_file);
160  ret = AVERROR(ENOENT);
161  goto err;
162  }
163 
164  // Read a single line (fgets stops at new line mark).
165  if (!fgets(c->gateway_buffer, sizeof(c->gateway_buffer) - 1, gateway_file)) {
166  av_log(h, AV_LOG_WARNING, "Unable to read from file (full uri: %s).\n",
167  ipfs_gateway_file);
168  ret = AVERROR(ENOENT);
169  goto err;
170  }
171 
172  // Replace first occurence of end of line with \0
173  c->gateway_buffer[strcspn(c->gateway_buffer, "\r\n")] = 0;
174 
175  // If strlen finds anything longer then 0 characters then we have a
176  // potential gateway url.
177  if (*c->gateway_buffer == '\0') {
179  "The IPFS gateway file (full uri: %s) appears to be empty. "
180  "Is the gateway started?\n",
181  ipfs_gateway_file);
182  ret = AVERROR(EILSEQ);
183  goto err;
184  } else {
185  // We're done, the c->gateway_buffer has something that looks valid.
186  ret = 1;
187  goto err;
188  }
189 
190 err:
191  if (gateway_file)
192  fclose(gateway_file);
193 
194  return ret;
195 }
196 
197 static int translate_ipfs_to_http(URLContext *h, const char *uri, int flags, AVDictionary **options)
198 {
199  const char *ipfs_cid;
200  char *fulluri = NULL;
201  int ret;
202  IPFSGatewayContext *c = h->priv_data;
203 
204  // Test for ipfs://, ipfs:, ipns:// and ipns:. This prefix is stripped from
205  // the string leaving just the CID in ipfs_cid.
206  int is_ipfs = av_stristart(uri, "ipfs://", &ipfs_cid);
207  int is_ipns = av_stristart(uri, "ipns://", &ipfs_cid);
208 
209  // We must have either ipns or ipfs.
210  if (!is_ipfs && !is_ipns) {
211  ret = AVERROR(EINVAL);
212  av_log(h, AV_LOG_WARNING, "Unsupported url %s\n", uri);
213  goto err;
214  }
215 
216  // If the CID has a length greater then 0 then we assume we have a proper working one.
217  // It could still be wrong but in that case the gateway should save us and
218  // ruturn a 403 error. The http protocol handles this.
219  if (strlen(ipfs_cid) < 1) {
220  av_log(h, AV_LOG_WARNING, "A CID must be provided.\n");
221  ret = AVERROR(EILSEQ);
222  goto err;
223  }
224 
225  // Populate c->gateway_buffer with whatever is in c->gateway
226  if (c->gateway != NULL) {
227  if (snprintf(c->gateway_buffer, sizeof(c->gateway_buffer), "%s",
228  c->gateway)
229  >= sizeof(c->gateway_buffer)) {
231  "The -gateway parameter is too long. "
232  "We allow a max of %zu characters\n",
233  sizeof(c->gateway_buffer));
234  ret = AVERROR(EINVAL);
235  goto err;
236  }
237  } else {
238  // Populate the IPFS gateway if we have any.
239  // If not, inform the user how to properly set one.
241 
242  if (ret < 1) {
243  // We fallback on dweb.link (managed by Protocol Labs).
244  snprintf(c->gateway_buffer, sizeof(c->gateway_buffer), "https://dweb.link");
245 
247  "IPFS does not appear to be running. "
248  "You’re now using the public gateway at dweb.link.\n");
250  "Installing IPFS locally is recommended to "
251  "improve performance and reliability, "
252  "and not share all your activity with a single IPFS gateway.\n"
253  "There are multiple options to define this gateway.\n"
254  "1. Call ffmpeg with a gateway param, "
255  "without a trailing slash: -gateway <url>.\n"
256  "2. Define an $IPFS_GATEWAY environment variable with the "
257  "full HTTP URL to the gateway "
258  "without trailing forward slash.\n"
259  "3. Define an $IPFS_PATH environment variable "
260  "and point it to the IPFS data path "
261  "- this is typically ~/.ipfs\n");
262  }
263  }
264 
265  // Test if the gateway starts with either http:// or https://
266  if (av_stristart(c->gateway_buffer, "http://", NULL) == 0
267  && av_stristart(c->gateway_buffer, "https://", NULL) == 0) {
269  "The gateway URL didn't start with http:// or "
270  "https:// and is therefore invalid.\n");
271  ret = AVERROR(EILSEQ);
272  goto err;
273  }
274 
275  // Concatenate the url.
276  // This ends up with something like: http://localhost:8080/ipfs/Qm.....
277  // The format of "%s%s%s%s" is the following:
278  // 1st %s = The gateway.
279  // 2nd %s = If the gateway didn't end in a slash, add a "/". Otherwise it's an empty string
280  // 3rd %s = Either ipns/ or ipfs/.
281  // 4th %s = The IPFS CID (Qm..., bafy..., ...).
282  fulluri = av_asprintf("%s%s%s%s",
283  c->gateway_buffer,
284  (c->gateway_buffer[strlen(c->gateway_buffer) - 1] == '/') ? "" : "/",
285  (is_ipns) ? "ipns/" : "ipfs/",
286  ipfs_cid);
287 
288  if (!fulluri) {
289  av_log(h, AV_LOG_ERROR, "Failed to compose the URL\n");
290  ret = AVERROR(ENOMEM);
291  goto err;
292  }
293 
294  // Pass the URL back to FFMpeg's protocol handler.
295  ret = ffurl_open_whitelist(&c->inner, fulluri, flags,
296  &h->interrupt_callback, options,
297  h->protocol_whitelist,
298  h->protocol_blacklist, h);
299  if (ret < 0) {
300  av_log(h, AV_LOG_WARNING, "Unable to open resource: %s\n", fulluri);
301  goto err;
302  }
303 
304 err:
305  av_free(fulluri);
306  return ret;
307 }
308 
309 static int ipfs_read(URLContext *h, unsigned char *buf, int size)
310 {
311  IPFSGatewayContext *c = h->priv_data;
312  return ffurl_read(c->inner, buf, size);
313 }
314 
315 static int64_t ipfs_seek(URLContext *h, int64_t pos, int whence)
316 {
317  IPFSGatewayContext *c = h->priv_data;
318  return ffurl_seek(c->inner, pos, whence);
319 }
320 
321 static int ipfs_close(URLContext *h)
322 {
323  IPFSGatewayContext *c = h->priv_data;
324  return ffurl_closep(&c->inner);
325 }
326 
327 #define OFFSET(x) offsetof(IPFSGatewayContext, x)
328 
329 static const AVOption options[] = {
330  {"gateway", "The gateway to ask for IPFS data.", OFFSET(gateway), AV_OPT_TYPE_STRING, {.str = NULL}, 0, 0, AV_OPT_FLAG_DECODING_PARAM},
331  {NULL},
332 };
333 
334 static const AVClass ipfs_context_class = {
335  .class_name = "IPFS",
336  .item_name = av_default_item_name,
337  .option = options,
338  .version = LIBAVUTIL_VERSION_INT,
339 };
340 
342  .name = "ipfs",
343  .url_open2 = translate_ipfs_to_http,
344  .url_read = ipfs_read,
345  .url_seek = ipfs_seek,
346  .url_close = ipfs_close,
347  .priv_data_size = sizeof(IPFSGatewayContext),
348  .priv_data_class = &ipfs_context_class,
349 };
350 
352  .name = "ipns",
353  .url_open2 = translate_ipfs_to_http,
354  .url_read = ipfs_read,
355  .url_seek = ipfs_seek,
356  .url_close = ipfs_close,
357  .priv_data_size = sizeof(IPFSGatewayContext),
358  .priv_data_class = &ipfs_context_class,
359 };
AV_LOG_WARNING
#define AV_LOG_WARNING
Something somehow does not look correct.
Definition: log.h:186
ff_ipfs_protocol
const URLProtocol ff_ipfs_protocol
Definition: ipfsgateway.c:341
AVERROR
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a all references to both lists are replaced with a reference to the intersection And when a single format is eventually chosen for a link amongst the remaining all references to the list are updated That means that if a filter requires that its input and output have the same format amongst a supported all it has to do is use a reference to the same list of formats query_formats can leave some formats unset and return AVERROR(EAGAIN) to cause the negotiation mechanism toagain later. That can be used by filters with complex requirements to use the format negotiated on one link to set the formats supported on another. Frame references ownership and permissions
opt.h
ffurl_seek
int64_t ffurl_seek(URLContext *h, int64_t pos, int whence)
Change the position that will be used by the next read/write operation on the resource accessed by h.
Definition: avio.c:428
av_asprintf
char * av_asprintf(const char *fmt,...)
Definition: avstring.c:116
IPFSGatewayContext
Definition: ipfsgateway.c:35
AVOption
AVOption.
Definition: opt.h:251
ff_ipns_protocol
const URLProtocol ff_ipns_protocol
Definition: ipfsgateway.c:351
freeenv_utf8
static void freeenv_utf8(char *var)
Definition: getenv_utf8.h:72
AVDictionary
Definition: dict.c:30
URLProtocol
Definition: url.h:53
os_support.h
AV_LOG_ERROR
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:180
ffurl_open_whitelist
int ffurl_open_whitelist(URLContext **puc, const char *filename, int flags, const AVIOInterruptCB *int_cb, AVDictionary **options, const char *whitelist, const char *blacklist, URLContext *parent)
Create an URLContext for accessing to the resource indicated by url, and open it.
Definition: avio.c:306
IPFSGatewayContext::gateway_buffer
char gateway_buffer[PATH_MAX]
Definition: ipfsgateway.c:43
AV_LOG_DEBUG
#define AV_LOG_DEBUG
Stuff which is only useful for libav* developers.
Definition: log.h:201
av_stristart
int av_stristart(const char *str, const char *pfx, const char **ptr)
Return non-zero if pfx is a prefix of str independent of case.
Definition: avstring.c:48
ipfs_seek
static int64_t ipfs_seek(URLContext *h, int64_t pos, int whence)
Definition: ipfsgateway.c:315
LIBAVUTIL_VERSION_INT
#define LIBAVUTIL_VERSION_INT
Definition: version.h:85
AVClass
Describe the class of an AVClass context structure.
Definition: log.h:66
NULL
#define NULL
Definition: coverity.c:32
OFFSET
#define OFFSET(x)
Definition: ipfsgateway.c:327
PATH_MAX
#define PATH_MAX
Definition: ipfsgateway.c:32
IPFSGatewayContext::gateway
char * gateway
Definition: ipfsgateway.c:39
av_default_item_name
const char * av_default_item_name(void *ptr)
Return the context name.
Definition: log.c:237
getenv_utf8
static char * getenv_utf8(const char *varname)
Definition: getenv_utf8.h:67
ipfs_context_class
static const AVClass ipfs_context_class
Definition: ipfsgateway.c:334
c
Undefined Behavior In the C some operations are like signed integer dereferencing freed accessing outside allocated Undefined Behavior must not occur in a C it is not safe even if the output of undefined operations is unused The unsafety may seem nit picking but Optimizing compilers have in fact optimized code on the assumption that no undefined Behavior occurs Optimizing code based on wrong assumptions can and has in some cases lead to effects beyond the output of computations The signed integer overflow problem in speed critical code Code which is highly optimized and works with signed integers sometimes has the problem that often the output of the computation does not c
Definition: undefined.txt:32
size
int size
Definition: twinvq_data.h:10344
URLProtocol::name
const char * name
Definition: url.h:54
getenv_utf8.h
AV_LOG_INFO
#define AV_LOG_INFO
Standard information.
Definition: log.h:191
URLContext
Definition: url.h:37
AV_OPT_FLAG_DECODING_PARAM
#define AV_OPT_FLAG_DECODING_PARAM
a generic parameter which can be set by the user for demuxing or decoding
Definition: opt.h:282
url.h
avpriv_fopen_utf8
FILE * avpriv_fopen_utf8(const char *path, const char *mode)
Open a file using a UTF-8 filename.
Definition: file_open.c:158
translate_ipfs_to_http
static int translate_ipfs_to_http(URLContext *h, const char *uri, int flags, AVDictionary **options)
Definition: ipfsgateway.c:197
ffurl_closep
int ffurl_closep(URLContext **hh)
Close the resource accessed by the URLContext h, and free the memory used by it.
Definition: avio.c:438
ret
ret
Definition: filter_design.txt:187
options
static const AVOption options[]
Definition: ipfsgateway.c:329
AVClass::class_name
const char * class_name
The name of the class; usually it is the same name as the context structure type to which the AVClass...
Definition: log.h:71
pos
unsigned int pos
Definition: spdifenc.c:412
ffurl_read
int ffurl_read(URLContext *h, unsigned char *buf, int size)
Read up to size bytes from the resource accessed by h, and store the read bytes in buf.
Definition: avio.c:401
IPFSGatewayContext::inner
URLContext * inner
Definition: ipfsgateway.c:37
av_free
#define av_free(p)
Definition: tableprint_vlc.h:33
flags
#define flags(name, subs,...)
Definition: cbs_av1.c:561
av_log
#define av_log(a,...)
Definition: tableprint_vlc.h:27
h
h
Definition: vp9dsp_template.c:2038
avstring.h
ipfs_read
static int ipfs_read(URLContext *h, unsigned char *buf, int size)
Definition: ipfsgateway.c:309
AV_OPT_TYPE_STRING
@ AV_OPT_TYPE_STRING
Definition: opt.h:229
ipfs_close
static int ipfs_close(URLContext *h)
Definition: ipfsgateway.c:321
snprintf
#define snprintf
Definition: snprintf.h:34
populate_ipfs_gateway
static int populate_ipfs_gateway(URLContext *h)
Definition: ipfsgateway.c:50