FFmpeg
ipfsgateway.c
Go to the documentation of this file.
1 /*
2  * IPFS and IPNS protocol support through IPFS Gateway.
3  * Copyright (c) 2022 Mark Gaiser
4  *
5  * This file is part of FFmpeg.
6  *
7  * FFmpeg is free software; you can redistribute it and/or
8  * modify it under the terms of the GNU Lesser General Public
9  * License as published by the Free Software Foundation; either
10  * version 2.1 of the License, or (at your option) any later version.
11  *
12  * FFmpeg is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15  * Lesser General Public License for more details.
16  *
17  * You should have received a copy of the GNU Lesser General Public
18  * License along with FFmpeg; if not, write to the Free Software
19  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20  */
21 
22 #include "libavutil/avstring.h"
23 #include "libavutil/file_open.h"
24 #include "libavutil/getenv_utf8.h"
25 #include "libavutil/opt.h"
26 #include <sys/stat.h>
27 #include "os_support.h"
28 #include "url.h"
29 
30 // Define the posix PATH_MAX if not there already.
31 // This fixes a compile issue for MSVC.
32 #ifndef PATH_MAX
33 #define PATH_MAX 4096
34 #endif
35 
36 typedef struct IPFSGatewayContext {
37  AVClass *class;
39  // Is filled by the -gateway argument and not changed after.
40  char *gateway;
41  // If the above gateway is non null, it will be copied into this buffer.
42  // Else this buffer will contain the auto detected gateway.
43  // In either case, the gateway to use will be in this buffer.
46 
47 // A best-effort way to find the IPFS gateway.
48 // Only the most appropiate gateway is set. It's not actually requested
49 // (http call) to prevent a potential slowdown in startup. A potential timeout
50 // is handled by the HTTP protocol.
52 {
53  IPFSGatewayContext *c = h->priv_data;
54  char ipfs_full_data_folder[PATH_MAX];
55  char ipfs_gateway_file[PATH_MAX];
56  struct stat st;
57  int stat_ret = 0;
58  int ret = AVERROR(EINVAL);
59  FILE *gateway_file = NULL;
60  char *env_ipfs_gateway, *env_ipfs_path;
61 
62  // Test $IPFS_GATEWAY.
63  env_ipfs_gateway = getenv_utf8("IPFS_GATEWAY");
64  if (env_ipfs_gateway != NULL) {
65  int printed = snprintf(c->gateway_buffer, sizeof(c->gateway_buffer),
66  "%s", env_ipfs_gateway);
67  freeenv_utf8(env_ipfs_gateway);
68  if (printed >= sizeof(c->gateway_buffer)) {
70  "The IPFS_GATEWAY environment variable "
71  "exceeds the maximum length. "
72  "We allow a max of %zu characters\n",
73  sizeof(c->gateway_buffer));
74  ret = AVERROR(EINVAL);
75  goto err;
76  }
77 
78  ret = 1;
79  goto err;
80  } else
81  av_log(h, AV_LOG_DEBUG, "$IPFS_GATEWAY is empty.\n");
82 
83  // We need to know the IPFS folder to - eventually - read the contents of
84  // the "gateway" file which would tell us the gateway to use.
85  env_ipfs_path = getenv_utf8("IPFS_PATH");
86  if (env_ipfs_path == NULL) {
87  int printed;
88  char *env_home = getenv_utf8("HOME");
89 
90  av_log(h, AV_LOG_DEBUG, "$IPFS_PATH is empty.\n");
91 
92  // Try via the home folder.
93  if (env_home == NULL) {
94  av_log(h, AV_LOG_WARNING, "$HOME appears to be empty.\n");
95  ret = AVERROR(EINVAL);
96  goto err;
97  }
98 
99  // Verify the composed path fits.
100  printed = snprintf(
101  ipfs_full_data_folder, sizeof(ipfs_full_data_folder),
102  "%s/.ipfs/", env_home);
103  freeenv_utf8(env_home);
104  if (printed >= sizeof(ipfs_full_data_folder)) {
106  "The IPFS data path exceeds the "
107  "max path length (%zu)\n",
108  sizeof(ipfs_full_data_folder));
109  ret = AVERROR(EINVAL);
110  goto err;
111  }
112 
113  // Stat the folder.
114  // It should exist in a default IPFS setup when run as local user.
115  stat_ret = stat(ipfs_full_data_folder, &st);
116 
117  if (stat_ret < 0) {
119  "Unable to find IPFS folder. We tried:\n"
120  "- $IPFS_PATH, which was empty.\n"
121  "- $HOME/.ipfs (full uri: %s) which doesn't exist.\n",
122  ipfs_full_data_folder);
123  ret = AVERROR(ENOENT);
124  goto err;
125  }
126  } else {
127  int printed = snprintf(
128  ipfs_full_data_folder, sizeof(ipfs_full_data_folder),
129  "%s", env_ipfs_path);
130  freeenv_utf8(env_ipfs_path);
131  if (printed >= sizeof(ipfs_full_data_folder)) {
133  "The IPFS_PATH environment variable "
134  "exceeds the maximum length. "
135  "We allow a max of %zu characters\n",
136  sizeof(c->gateway_buffer));
137  ret = AVERROR(EINVAL);
138  goto err;
139  }
140  }
141 
142  // Copy the fully composed gateway path into ipfs_gateway_file.
143  if (snprintf(ipfs_gateway_file, sizeof(ipfs_gateway_file), "%sgateway",
144  ipfs_full_data_folder)
145  >= sizeof(ipfs_gateway_file)) {
147  "The IPFS gateway file path exceeds "
148  "the max path length (%zu)\n",
149  sizeof(ipfs_gateway_file));
150  ret = AVERROR(ENOENT);
151  goto err;
152  }
153 
154  // Get the contents of the gateway file.
155  gateway_file = avpriv_fopen_utf8(ipfs_gateway_file, "r");
156  if (!gateway_file) {
158  "The IPFS gateway file (full uri: %s) doesn't exist. "
159  "Is the gateway enabled?\n",
160  ipfs_gateway_file);
161  ret = AVERROR(ENOENT);
162  goto err;
163  }
164 
165  // Read a single line (fgets stops at new line mark).
166  if (!fgets(c->gateway_buffer, sizeof(c->gateway_buffer) - 1, gateway_file)) {
167  av_log(h, AV_LOG_WARNING, "Unable to read from file (full uri: %s).\n",
168  ipfs_gateway_file);
169  ret = AVERROR(ENOENT);
170  goto err;
171  }
172 
173  // Replace first occurence of end of line with \0
174  c->gateway_buffer[strcspn(c->gateway_buffer, "\r\n")] = 0;
175 
176  // If strlen finds anything longer then 0 characters then we have a
177  // potential gateway url.
178  if (*c->gateway_buffer == '\0') {
180  "The IPFS gateway file (full uri: %s) appears to be empty. "
181  "Is the gateway started?\n",
182  ipfs_gateway_file);
183  ret = AVERROR(EILSEQ);
184  goto err;
185  } else {
186  // We're done, the c->gateway_buffer has something that looks valid.
187  ret = 1;
188  goto err;
189  }
190 
191 err:
192  if (gateway_file)
193  fclose(gateway_file);
194 
195  return ret;
196 }
197 
198 static int translate_ipfs_to_http(URLContext *h, const char *uri, int flags, AVDictionary **options)
199 {
200  const char *ipfs_cid;
201  char *fulluri = NULL;
202  int ret;
203  IPFSGatewayContext *c = h->priv_data;
204 
205  // Test for ipfs://, ipfs:, ipns:// and ipns:. This prefix is stripped from
206  // the string leaving just the CID in ipfs_cid.
207  int is_ipfs = av_stristart(uri, "ipfs://", &ipfs_cid);
208  int is_ipns = av_stristart(uri, "ipns://", &ipfs_cid);
209 
210  // We must have either ipns or ipfs.
211  if (!is_ipfs && !is_ipns) {
212  ret = AVERROR(EINVAL);
213  av_log(h, AV_LOG_WARNING, "Unsupported url %s\n", uri);
214  goto err;
215  }
216 
217  // If the CID has a length greater then 0 then we assume we have a proper working one.
218  // It could still be wrong but in that case the gateway should save us and
219  // ruturn a 403 error. The http protocol handles this.
220  if (strlen(ipfs_cid) < 1) {
221  av_log(h, AV_LOG_WARNING, "A CID must be provided.\n");
222  ret = AVERROR(EILSEQ);
223  goto err;
224  }
225 
226  // Populate c->gateway_buffer with whatever is in c->gateway
227  if (c->gateway != NULL) {
228  if (snprintf(c->gateway_buffer, sizeof(c->gateway_buffer), "%s",
229  c->gateway)
230  >= sizeof(c->gateway_buffer)) {
232  "The -gateway parameter is too long. "
233  "We allow a max of %zu characters\n",
234  sizeof(c->gateway_buffer));
235  ret = AVERROR(EINVAL);
236  goto err;
237  }
238  } else {
239  // Populate the IPFS gateway if we have any.
240  // If not, inform the user how to properly set one.
242 
243  if (ret < 1) {
245  "IPFS does not appear to be running.\n\n"
246  "Installing IPFS locally is recommended to "
247  "improve performance and reliability, "
248  "and not share all your activity with a single IPFS gateway.\n"
249  "There are multiple options to define this gateway.\n"
250  "1. Call ffmpeg with a gateway param, "
251  "without a trailing slash: -gateway <url>.\n"
252  "2. Define an $IPFS_GATEWAY environment variable with the "
253  "full HTTP URL to the gateway "
254  "without trailing forward slash.\n"
255  "3. Define an $IPFS_PATH environment variable "
256  "and point it to the IPFS data path "
257  "- this is typically ~/.ipfs\n");
258  ret = AVERROR(EINVAL);
259  goto err;
260  }
261  }
262 
263  // Test if the gateway starts with either http:// or https://
264  if (av_stristart(c->gateway_buffer, "http://", NULL) == 0
265  && av_stristart(c->gateway_buffer, "https://", NULL) == 0) {
267  "The gateway URL didn't start with http:// or "
268  "https:// and is therefore invalid.\n");
269  ret = AVERROR(EILSEQ);
270  goto err;
271  }
272 
273  // Concatenate the url.
274  // This ends up with something like: http://localhost:8080/ipfs/Qm.....
275  // The format of "%s%s%s%s" is the following:
276  // 1st %s = The gateway.
277  // 2nd %s = If the gateway didn't end in a slash, add a "/". Otherwise it's an empty string
278  // 3rd %s = Either ipns/ or ipfs/.
279  // 4th %s = The IPFS CID (Qm..., bafy..., ...).
280  fulluri = av_asprintf("%s%s%s%s",
281  c->gateway_buffer,
282  (c->gateway_buffer[strlen(c->gateway_buffer) - 1] == '/') ? "" : "/",
283  (is_ipns) ? "ipns/" : "ipfs/",
284  ipfs_cid);
285 
286  if (!fulluri) {
287  av_log(h, AV_LOG_ERROR, "Failed to compose the URL\n");
288  ret = AVERROR(ENOMEM);
289  goto err;
290  }
291 
292  // Pass the URL back to FFMpeg's protocol handler.
293  ret = ffurl_open_whitelist(&c->inner, fulluri, flags,
294  &h->interrupt_callback, options,
295  h->protocol_whitelist,
296  h->protocol_blacklist, h);
297  if (ret < 0) {
298  av_log(h, AV_LOG_WARNING, "Unable to open resource: %s\n", fulluri);
299  goto err;
300  }
301 
302 err:
303  av_free(fulluri);
304  return ret;
305 }
306 
307 static int ipfs_read(URLContext *h, unsigned char *buf, int size)
308 {
309  IPFSGatewayContext *c = h->priv_data;
310  return ffurl_read(c->inner, buf, size);
311 }
312 
313 static int64_t ipfs_seek(URLContext *h, int64_t pos, int whence)
314 {
315  IPFSGatewayContext *c = h->priv_data;
316  return ffurl_seek(c->inner, pos, whence);
317 }
318 
319 static int ipfs_close(URLContext *h)
320 {
321  IPFSGatewayContext *c = h->priv_data;
322  return ffurl_closep(&c->inner);
323 }
324 
325 #define OFFSET(x) offsetof(IPFSGatewayContext, x)
326 
327 static const AVOption options[] = {
328  {"gateway", "The gateway to ask for IPFS data.", OFFSET(gateway), AV_OPT_TYPE_STRING, {.str = NULL}, 0, 0, AV_OPT_FLAG_DECODING_PARAM},
329  {NULL},
330 };
331 
333  .class_name = "IPFS Gateway",
334  .item_name = av_default_item_name,
335  .option = options,
336  .version = LIBAVUTIL_VERSION_INT,
337 };
338 
340  .name = "ipfs",
341  .url_open2 = translate_ipfs_to_http,
342  .url_read = ipfs_read,
343  .url_seek = ipfs_seek,
344  .url_close = ipfs_close,
345  .priv_data_size = sizeof(IPFSGatewayContext),
346  .priv_data_class = &ipfs_gateway_context_class,
347 };
348 
350  .name = "ipns",
351  .url_open2 = translate_ipfs_to_http,
352  .url_read = ipfs_read,
353  .url_seek = ipfs_seek,
354  .url_close = ipfs_close,
355  .priv_data_size = sizeof(IPFSGatewayContext),
356  .priv_data_class = &ipfs_gateway_context_class,
357 };
AV_LOG_WARNING
#define AV_LOG_WARNING
Something somehow does not look correct.
Definition: log.h:186
AVERROR
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a all references to both lists are replaced with a reference to the intersection And when a single format is eventually chosen for a link amongst the remaining all references to the list are updated That means that if a filter requires that its input and output have the same format amongst a supported all it has to do is use a reference to the same list of formats query_formats can leave some formats unset and return AVERROR(EAGAIN) to cause the negotiation mechanism toagain later. That can be used by filters with complex requirements to use the format negotiated on one link to set the formats supported on another. Frame references ownership and permissions
opt.h
ffurl_seek
int64_t ffurl_seek(URLContext *h, int64_t pos, int whence)
Change the position that will be used by the next read/write operation on the resource accessed by h.
Definition: avio.c:428
av_asprintf
char * av_asprintf(const char *fmt,...)
Definition: avstring.c:116
IPFSGatewayContext
Definition: ipfsgateway.c:36
AVOption
AVOption.
Definition: opt.h:251
freeenv_utf8
static void freeenv_utf8(char *var)
Definition: getenv_utf8.h:72
AVDictionary
Definition: dict.c:32
URLProtocol
Definition: url.h:53
os_support.h
AV_LOG_ERROR
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:180
ffurl_open_whitelist
int ffurl_open_whitelist(URLContext **puc, const char *filename, int flags, const AVIOInterruptCB *int_cb, AVDictionary **options, const char *whitelist, const char *blacklist, URLContext *parent)
Create an URLContext for accessing to the resource indicated by url, and open it.
Definition: avio.c:306
ff_ipns_gateway_protocol
const URLProtocol ff_ipns_gateway_protocol
Definition: ipfsgateway.c:349
IPFSGatewayContext::gateway_buffer
char gateway_buffer[PATH_MAX]
Definition: ipfsgateway.c:44
AV_LOG_DEBUG
#define AV_LOG_DEBUG
Stuff which is only useful for libav* developers.
Definition: log.h:201
av_stristart
int av_stristart(const char *str, const char *pfx, const char **ptr)
Return non-zero if pfx is a prefix of str independent of case.
Definition: avstring.c:48
file_open.h
ipfs_seek
static int64_t ipfs_seek(URLContext *h, int64_t pos, int whence)
Definition: ipfsgateway.c:313
LIBAVUTIL_VERSION_INT
#define LIBAVUTIL_VERSION_INT
Definition: version.h:85
AVClass
Describe the class of an AVClass context structure.
Definition: log.h:66
NULL
#define NULL
Definition: coverity.c:32
OFFSET
#define OFFSET(x)
Definition: ipfsgateway.c:325
PATH_MAX
#define PATH_MAX
Definition: ipfsgateway.c:33
IPFSGatewayContext::gateway
char * gateway
Definition: ipfsgateway.c:40
av_default_item_name
const char * av_default_item_name(void *ptr)
Return the context name.
Definition: log.c:237
getenv_utf8
static char * getenv_utf8(const char *varname)
Definition: getenv_utf8.h:67
c
Undefined Behavior In the C some operations are like signed integer dereferencing freed accessing outside allocated Undefined Behavior must not occur in a C it is not safe even if the output of undefined operations is unused The unsafety may seem nit picking but Optimizing compilers have in fact optimized code on the assumption that no undefined Behavior occurs Optimizing code based on wrong assumptions can and has in some cases lead to effects beyond the output of computations The signed integer overflow problem in speed critical code Code which is highly optimized and works with signed integers sometimes has the problem that often the output of the computation does not c
Definition: undefined.txt:32
ipfs_gateway_context_class
static const AVClass ipfs_gateway_context_class
Definition: ipfsgateway.c:332
size
int size
Definition: twinvq_data.h:10344
URLProtocol::name
const char * name
Definition: url.h:54
getenv_utf8.h
AV_LOG_INFO
#define AV_LOG_INFO
Standard information.
Definition: log.h:191
URLContext
Definition: url.h:37
ff_ipfs_gateway_protocol
const URLProtocol ff_ipfs_gateway_protocol
Definition: ipfsgateway.c:339
AV_OPT_FLAG_DECODING_PARAM
#define AV_OPT_FLAG_DECODING_PARAM
a generic parameter which can be set by the user for demuxing or decoding
Definition: opt.h:282
url.h
avpriv_fopen_utf8
FILE * avpriv_fopen_utf8(const char *path, const char *mode)
Open a file using a UTF-8 filename.
Definition: file_open.c:159
translate_ipfs_to_http
static int translate_ipfs_to_http(URLContext *h, const char *uri, int flags, AVDictionary **options)
Definition: ipfsgateway.c:198
ffurl_closep
int ffurl_closep(URLContext **hh)
Close the resource accessed by the URLContext h, and free the memory used by it.
Definition: avio.c:438
ret
ret
Definition: filter_design.txt:187
options
static const AVOption options[]
Definition: ipfsgateway.c:327
AVClass::class_name
const char * class_name
The name of the class; usually it is the same name as the context structure type to which the AVClass...
Definition: log.h:71
pos
unsigned int pos
Definition: spdifenc.c:413
ffurl_read
int ffurl_read(URLContext *h, unsigned char *buf, int size)
Read up to size bytes from the resource accessed by h, and store the read bytes in buf.
Definition: avio.c:401
IPFSGatewayContext::inner
URLContext * inner
Definition: ipfsgateway.c:38
av_free
#define av_free(p)
Definition: tableprint_vlc.h:33
flags
#define flags(name, subs,...)
Definition: cbs_av1.c:561
av_log
#define av_log(a,...)
Definition: tableprint_vlc.h:27
h
h
Definition: vp9dsp_template.c:2038
avstring.h
ipfs_read
static int ipfs_read(URLContext *h, unsigned char *buf, int size)
Definition: ipfsgateway.c:307
AV_OPT_TYPE_STRING
@ AV_OPT_TYPE_STRING
Definition: opt.h:229
ipfs_close
static int ipfs_close(URLContext *h)
Definition: ipfsgateway.c:319
snprintf
#define snprintf
Definition: snprintf.h:34
populate_ipfs_gateway
static int populate_ipfs_gateway(URLContext *h)
Definition: ipfsgateway.c:51