diff options
author | Sebastian Dröge <sebastian.droege@collabora.co.uk> | 2012-10-25 13:21:13 +0200 |
---|---|---|
committer | Sebastian Dröge <sebastian.droege@collabora.co.uk> | 2012-10-25 13:21:13 +0200 |
commit | 8421cfdca94795cdb6513d3d6c38abd3a416373b (patch) | |
tree | b259bfa95d54a3dbc09033dc5040c6e27a0f1036 /docs/pwg | |
parent | f4e033e836d36ee95103886032932bf823db5e3f (diff) |
Imported Upstream version 1.0.2upstream/1.0.2
Diffstat (limited to 'docs/pwg')
-rw-r--r-- | docs/pwg/Makefile.in | 6 | ||||
-rw-r--r-- | docs/pwg/advanced-clock.xml | 30 | ||||
-rw-r--r-- | docs/pwg/advanced-events.xml | 8 | ||||
-rw-r--r-- | docs/pwg/advanced-negotiation.xml | 624 | ||||
-rw-r--r-- | docs/pwg/advanced-qos.xml | 283 | ||||
-rw-r--r-- | docs/pwg/building-boiler.xml | 10 | ||||
-rw-r--r-- | docs/pwg/building-eventfn.xml | 16 | ||||
-rw-r--r-- | docs/pwg/building-queryfn.xml | 72 | ||||
-rw-r--r-- | docs/pwg/intro-basics.xml | 4 | ||||
-rw-r--r-- | docs/pwg/pwg.xml | 16 |
10 files changed, 778 insertions, 291 deletions
diff --git a/docs/pwg/Makefile.in b/docs/pwg/Makefile.in index 6919b13..473b2cb 100644 --- a/docs/pwg/Makefile.in +++ b/docs/pwg/Makefile.in @@ -105,7 +105,7 @@ am__aclocal_m4_deps = $(top_srcdir)/common/m4/as-ac-expand.m4 \ $(top_srcdir)/common/m4/gst.m4 \ $(top_srcdir)/common/m4/gtk-doc.m4 \ $(top_srcdir)/common/m4/introspection.m4 \ - $(top_srcdir)/common/m4/pkg.m4 \ + $(top_srcdir)/common/m4/pkg.m4 $(top_srcdir)/m4/ax_pthread.m4 \ $(top_srcdir)/m4/check-checks.m4 $(top_srcdir)/m4/gettext.m4 \ $(top_srcdir)/m4/iconv.m4 $(top_srcdir)/m4/intlmacosx.m4 \ $(top_srcdir)/m4/lib-ld.m4 $(top_srcdir)/m4/lib-link.m4 \ @@ -353,6 +353,9 @@ PKG_CONFIG = @PKG_CONFIG@ PLUGINDIR = @PLUGINDIR@ POSUB = @POSUB@ PROFILE_CFLAGS = @PROFILE_CFLAGS@ +PTHREAD_CC = @PTHREAD_CC@ +PTHREAD_CFLAGS = @PTHREAD_CFLAGS@ +PTHREAD_LIBS = @PTHREAD_LIBS@ PYTHON = @PYTHON@ PYTHON_EXEC_PREFIX = @PYTHON_EXEC_PREFIX@ PYTHON_PLATFORM = @PYTHON_PLATFORM@ @@ -389,6 +392,7 @@ am__leading_dot = @am__leading_dot@ am__quote = @am__quote@ am__tar = @am__tar@ am__untar = @am__untar@ +ax_pthread_config = @ax_pthread_config@ bindir = @bindir@ build = @build@ build_alias = @build_alias@ diff --git a/docs/pwg/advanced-clock.xml b/docs/pwg/advanced-clock.xml index beb6bd7..89c1f29 100644 --- a/docs/pwg/advanced-clock.xml +++ b/docs/pwg/advanced-clock.xml @@ -73,7 +73,10 @@ <para> Synchronization is now a matter of making sure that a buffer with a certain running-time is played when the clock reaches the same - running-time. Usually this task is done by sink elements. + running-time. Usually this task is done by sink elements. Sink also + have to take into account the latency configured in the pipeline and + add this to the buffer running-time before synchronizing to the + pipeline clock. </para> </sect1> @@ -120,11 +123,11 @@ </sect2> <sect2> - <title>Parser elements </title> + <title>Parser/Decoder/Encoder elements </title> <para> - Parser elements must use the incomming timestamps and transfer those - to the resulting output buffers. They are allowed to interpolate or - reconstruct timestamps on missing input buffers when they can. + Parser/Decoder elements must use the incomming timestamps and transfer + those to the resulting output buffers. They are allowed to interpolate + or reconstruct timestamps on missing input buffers when they can. </para> </sect2> @@ -139,8 +142,18 @@ buffer timestamps. </para> </sect2> + + <sect2> + <title>Muxer elements</title> + <para> + Muxer elements should use the incomming buffer running-time to mux the + different streams together. They should copy the incomming running-time + to the outgoing buffers. + </para> + </sect2> - <sect2> <title> Sink elements </title> + <sect2> + <title>Sink elements</title> <para> If the element is intended to emit samples at a specific time (real time playing), the element should require a clock, and thus implement the @@ -148,8 +161,9 @@ </para> <para> The sink should then make sure that the sample with running-time is played - exactly when the pipeline clock reaches that running-time. Some elements - might use the clock API such as <function>gst_clock_id_wait()</function> + exactly when the pipeline clock reaches that running-time + latency. + Some elements might use the clock API such as + <function>gst_clock_id_wait()</function> to perform this action. Other sinks might need to use other means of scheduling timely playback of the data. </para> diff --git a/docs/pwg/advanced-events.xml b/docs/pwg/advanced-events.xml index 5d10d55..997d5b3 100644 --- a/docs/pwg/advanced-events.xml +++ b/docs/pwg/advanced-events.xml @@ -200,7 +200,9 @@ gst_my_filter_sink_event (GstPad *pad, GstObject * parent, GstEvent * event) <sect2 id="section-events-caps" xreflabel="Caps"> <title>Caps</title> <para> - WRITEME + The CAPS event contains the format description of the following + buffers. See <xref linkend="chapter-negotiation"/> for more + information about negotiation. </para> </sect2> @@ -378,7 +380,9 @@ gst_my_filter_sink_event (GstPad *pad, GstObject * parent, GstEvent * event) <sect2 id="section-events-qos" xreflabel="Quality Of Service (QOS)"> <title>Quality Of Service (QOS)</title> <para> - WRITEME + The QOS event contains a report about the current real-time + performance of the stream. See more info in + <xref linkend="chapter-advanced-qos"/>. </para> </sect2> diff --git a/docs/pwg/advanced-negotiation.xml b/docs/pwg/advanced-negotiation.xml index 6264ec2..bc0ecbb 100644 --- a/docs/pwg/advanced-negotiation.xml +++ b/docs/pwg/advanced-negotiation.xml @@ -1,185 +1,262 @@ <chapter id="chapter-negotiation" xreflabel="Caps negotiation"> <title>Caps negotiation</title> <para> - Caps negotiation is the process where elements configure themselves - and each other for streaming a particular media format over their pads. - Since different types of elements have different requirements for the - media formats they can negotiate to, it is important that this process - is generic and implements all those use cases correctly. - </para> - <para> - In this chapter, we will discuss downstream negotiation and upstream - negotiation from a pipeline perspective, implicating the responsibilities - of different types of elements in a pipeline, and we will introduce the - concept of <emphasis>fixed caps</emphasis>. + Caps negotiation is the act of finding a media format (GstCaps) between + elements that they can handle. This process in &GStreamer; can in most + cases find an optimal solution for the complete pipeline. In this section + we explain how this works. </para> - <sect1 id="section-nego-requirements" xreflabel="Caps negotiation use cases"> - <title>Caps negotiation use cases</title> - <para> - Let's take the case of a file source, linked to a demuxer, linked to a - decoder, linked to a converter with a caps filter and finally an audio - output. When data flow originally starts, the demuxer will parse the - file header (e.g. the Ogg headers), and notice that there is, for - example, a Vorbis stream in this Ogg file. Noticing that, it will - create an output pad for the Vorbis elementary stream and set a - Vorbis-caps on it. Lastly, it adds the pad. As of this point, the pad - is ready to be used to stream data, and so the Ogg demuxer is now done. - This pad is <emphasis>not</emphasis> re-negotiable, since the type of - the data stream is embedded within the data. - </para> - <para> - The Vorbis decoder will decode the Vorbis headers and the Vorbis data - coming in on its sinkpad. Now, some decoders may be able to output in - multiple output formats, for example both 16-bit integer output and - floating-point output, whereas other decoders may be able to only decode - into one specific format, e.g. only floating-point (32-bit) audio. Those - two cases have consequences for how caps negotiation should be - implemented in this decoder element. In the one case, it is possible to - use fixed caps, and you're done. In the other case, however, you should - implement the possibility for <emphasis>renegotiation</emphasis> in this - element, which is the possibility for the data format to be changed to - another format at some point in the future. We will discuss how to do - this in one of the sections further on in this chapter. - </para> + <sect1 id="section-nego-basics"> + <title>Caps negotiation basics</title> <para> - The filter can be used by applications to force, for example, a specific - channel configuration (5.1/surround or 2.0/stereo), on the pipeline, so - that the user can enjoy sound coming from all its speakers. The audio - sink, in this example, is a standard ALSA output element (alsasink). - The converter element supports any-to-any, and the filter will make sure - that only a specifically wanted channel configuration streams through - this link (as provided by the user's channel configuration preference). - By changing this preference while the pipeline is running, some elements - will have to renegotiate <emphasis>while the pipeline is - running</emphasis>. This is done through upstream caps renegotiation. - That, too, will be discussed in detail in a section further below. + In &GStreamer;, negotiation of the media format always follows the + following simple rules: </para> + <itemizedlist> + <listitem> + <para> + A downstream element suggest a format on its sinkpad and places the + suggestion in the result of the CAPS query performed on the sinkpad. + See also <xref linkend="section-nego-getcaps"/>. + </para> + </listitem> + <listitem> + <para> + An upstream element decides on a format. It sends the selected media + format downstream on its source pad with a CAPS event. Downstream + elements reconfigure themselves to handle the media type in the CAPS + event on the sinkpad. + </para> + </listitem> + <listitem> + <para> + An upstream element can inform downstream that it would like to + suggest a new format by sending a RECONFIGURE event upstream. The + RECONFIGURE event simply instructs an upstream element to restart + the negotiation phase. Because the element that sent out the + RECONFIGURE event is now suggesting another format, the format + in the pipeline might change. + </para> + </listitem> + </itemizedlist> <para> - In order for caps negotiation on non-fixed links to work correctly, - pads can optionally implement a query function that tells peer elements - what formats it supports and/or prefers. When upstream renegotiation is - triggered, this becomes important. + In addition to the CAPS and RECONFIGURE event and the CAPS query, there + is an ACCEPT_CAPS query to quickly check if a certain caps can + be accepted by an element. </para> <para> - Downstream elements are notified of a newly set caps with a - GST_EVENT_CAPS on the sinkpad. So when the vorbis decoder sets a caps on - its source pad (to configure the output format), the converter will - receive a caps event. - When an element receives a buffer, it should check if it has received - all needed format information in a CAPS event previously. If it hasn't, - it should return an error from the chain function. + All negotiation follows these simple rules. Let's take a look at some + typical uses cases and how negotiation happens. </para> </sect1> - <sect1 id="section-nego-fixedcaps" xreflabel="Fixed caps"> - <title>Fixed caps</title> - <para> - The simplest way in which to do caps negotiation is setting a fixed - caps on a pad. After a fixed caps has been set, the pad can not be - renegotiated from the outside. The only way to reconfigure the pad - is for the element owning the pad to set a new fixed caps on the pad. - Fixed caps is a setup property for pads, called when creating the pad: - </para> - <programlisting> -[..] - pad = gst_pad_new_from_static_template (..); - gst_pad_use_fixed_caps (pad); -[..] - </programlisting> + <sect1 id="section-nego-usecases"> + <title>Caps negotiation use cases</title> <para> - The fixed caps can then be set on the pad by calling - <function>gst_pad_set_caps ()</function>. + In what follows we will look at some use cases for push-mode scheduling. + The pull-mode scheduling negotiation phase is discussed in + <xref linkend="section-nego-pullmode"/> and is actually similar as we + will see. </para> - <programlisting> -[..] - caps = gst_caps_new_simple ("audio/x-raw", - "format", G_TYPE_STRING, GST_AUDIO_NE(F32), - "rate", G_TYPE_INT, <samplerate>, - "channels", G_TYPE_INT, <num-channels>, NULL); - if (!gst_pad_set_caps (pad, caps)) { - GST_ELEMENT_ERROR (element, CORE, NEGOTIATION, (NULL), - ("Some debug information here")); - return GST_FLOW_ERROR; - } -[..] - </programlisting> <para> - Elements that could implement fixed caps (on their source pads) are, - in general, all elements that are not renegotiable. Examples include: + Since the sink pads only suggest formats and the source pads need to + decide, the most complicated work is done in the source pads. + We can identify 3 caps negotiation use cases for the source pads: </para> <itemizedlist> <listitem> <para> - A typefinder, since the type found is part of the actual data stream - and can thus not be re-negotiated. + Fixed negotiation. An element can output one format only. + See <xref linkend="section-nego-fixed"/>. </para> </listitem> <listitem> <para> - Pretty much all demuxers, since the contained elementary data - streams are defined in the file headers, and thus not - renegotiable. + Transform negotiation. There is a (fixed) transform between the + input and output format of the element, usually based on some + element property. The caps that the element will produce depend + on the upstream caps and the caps that the element can accept + depend on the downstream caps. + See <xref linkend="section-nego-transform"/>. </para> </listitem> <listitem> <para> - Some decoders, where the format is embedded in the data stream - and not part of the peercaps <emphasis>and</emphasis> where the - decoder itself is not reconfigurable, too. + Dynamic negotiation. An element can output many formats. + See <xref linkend="section-nego-dynamic"/>. </para> </listitem> </itemizedlist> - <para> - All other elements that need to be configured for the format should - implement full caps negotiation, which will be explained in the next - few sections. - </para> - </sect1> - <sect1 id="section-nego-downstream" xreflabel="Downstream caps negotiation"> - <title>Downstream caps negotiation</title> - <para> - Downstream negotiation takes place when a format needs to be set on a - source pad to configure the output format, but this element allows - renegotiation because its format is configured on the sinkpad caps, - or because it supports multiple formats. The requirements for doing - the actual negotiation differ slightly. - </para> + <sect2 id="section-nego-fixed"> + <title>Fixed negotiation</title> + <para> + In this case, the source pad can only produce a fixed format. Usually + this format is encoded inside the media. No downstream element can + ask for a different format, the only way that the source pad will + renegotiate is when the element decides to change the caps itself. + </para> + <para> + Elements that could implement fixed caps (on their source pads) are, + in general, all elements that are not renegotiable. Examples include: + </para> + <itemizedlist> + <listitem> + <para> + A typefinder, since the type found is part of the actual data stream + and can thus not be re-negotiated. The typefinder will look at the + stream of bytes, figure out the type, send a CAPS event with the + caps and then push buffers of the type. + </para> + </listitem> + <listitem> + <para> + Pretty much all demuxers, since the contained elementary data + streams are defined in the file headers, and thus not + renegotiable. + </para> + </listitem> + <listitem> + <para> + Some decoders, where the format is embedded in the data stream + and not part of the peercaps <emphasis>and</emphasis> where the + decoder itself is not reconfigurable, too. + </para> + </listitem> + <listitem> + <para> + Some sources that produce a fixed format. + </para> + </listitem> + </itemizedlist> + <para> + <function>gst_pad_use_fixed_caps()</function> is used on the source + pad with fixed caps. As long as the pad is not negotiated, the default + CAPS query will return the caps presented in the padtemplate. As soon + as the pad is negotiated, the CAPS query will return the negotiated + caps (and nothing else). These are the relevant code snippets for fixed + caps source pads. + </para> + <programlisting> +<![CDATA[ +[..] + pad = gst_pad_new_from_static_template (..); + gst_pad_use_fixed_caps (pad); +[..] +]]> + </programlisting> + <para> + The fixed caps can then be set on the pad by calling + <function>gst_pad_set_caps ()</function>. + </para> + <programlisting> +<![CDATA[ +[..] + caps = gst_caps_new_simple ("audio/x-raw", + "format", G_TYPE_STRING, GST_AUDIO_NE(F32), + "rate", G_TYPE_INT, <samplerate>, + "channels", G_TYPE_INT, <num-channels>, NULL); + if (!gst_pad_set_caps (pad, caps)) { + GST_ELEMENT_ERROR (element, CORE, NEGOTIATION, (NULL), + ("Some debug information here")); + return GST_FLOW_ERROR; + } +[..] +]]> + </programlisting> + <para> + These types of elements also don't have a relation between the input + format and the output format, the input caps simply don't contain the + information needed to produce the output caps. + </para> + <para> + All other elements that need to be configured for the format should + implement full caps negotiation, which will be explained in the next + few sections. + </para> + </sect2> - <sect2 id="section-nego-downstream-embed" - xreflabel="Negotiating caps embedded in input caps"> - <title>Negotiating caps embedded in input caps</title> + <sect2 id="section-nego-transform"> + <title>Transform negotiation</title> + <para> + In this negotiation technique, there is a fixed transform between + the element input caps and the output caps. This transformation + could be parameterized by element properties but not by the + content of the stream (see <xref linkend="section-nego-fixed"/> + for that use-case). + </para> <para> - Many elements, particularly effects and converters, will be able - to parse the format of the stream from their input caps, and decide - the output format right at that time already. For those elements, all - (downstream) caps negotiation can be done from the - <function>_event ()</function> function when a GST_EVENT_CAPS is - received on the sinkpad. This CAPS event is received whenever the - format changes or when no format was negotiated yet. It will always - be called before you receive the buffer in the format specified in - the CAPS event. + The caps that the element can accept depend on the (fixed + transformation) downstream caps. The caps that the element can + produce depend on the (fixed transformation of) the upstream + caps. </para> <para> - In the <function>_event ()</function>-function, the element can - forward the CAPS event to the next element and, if that pad accepts the - format too, the element can parse the relevant parameters from the - caps and configure itself internally. The caps passed to this function - is <emphasis>always</emphasis> a subset of the template caps, so - there's no need for extensive safety checking. The following example - should give a clear indication of how such a function can be - implemented: + This type of element can usually set caps on its source pad from + the <function>_event()</function> function on the sink pad when + it received the CAPS event. This means that the caps transform + function transforms a fixed caps into another fixed caps. + Examples of elements include: </para> - <programlisting><!-- example-begin forwardcaps.c a --><!-- -#include "init.func" -static GstCaps * -gst_my_filter_getcaps (GstPad * pad) + <itemizedlist> + <listitem> + <para> + Videobox. It adds configurable border around a video frame + depending on object properties. + </para> + </listitem> + <listitem> + <para> + Identity elements. All elements that don't change the format + of the data, only the content. Video and audio effects are an + example. Other examples include elements that inspect the + stream. + </para> + </listitem> + <listitem> + <para> + Some decoders and encoders, where the output format is defined + by input format, like mulawdec and mulawenc. These decoders + usually have no headers that define the content of the stream. + They are usually more like conversion elements. + </para> + </listitem> + </itemizedlist> + <para> + Below is an example of a negotiation steps of a typical transform + element. In the sink pad CAPS event handler, we compute the caps + for the source pad and set those. + </para> + <programlisting> +<![CDATA[ + [...] + +static gboolean +gst_my_filter_setcaps (GstMyFilter *filter, + GstCaps *caps) { - return NULL; + GstStructure *structure; + int rate, channels; + gboolean ret; + GstCaps *outcaps; + + structure = gst_caps_get_structure (caps, 0); + ret = gst_structure_get_int (structure, "rate", &rate); + ret = ret && gst_structure_get_int (structure, "channels", &channels); + if (!ret) + return FALSE; + + outcaps = gst_caps_new_simple ("audio/x-raw", + "format", G_TYPE_STRING, GST_AUDIO_NE(S16), + "rate", G_TYPE_INT, samplerate, + "channels", G_TYPE_INT, channels, NULL); + ret = gst_pad_set_caps (filter->srcpad, outcaps); + gst_caps_unref (outcaps); + + return ret; } ---><!-- example-end forwardcaps.c a --> -<!-- example-begin forwardcaps.c b --> + static gboolean gst_my_filter_sink_event (GstPad *pad, GstObject *parent, @@ -192,19 +269,9 @@ gst_my_filter_sink_event (GstPad *pad, case GST_EVENT_CAPS: { GstCaps *caps; - GstStructure *s; - gst_event_parse_caps (event, &caps); - - /* forward-negotiate */ - ret = gst_pad_set_caps (filter->srcpad, caps); - if (!ret) - return FALSE; - - /* negotiation succeeded, so now configure ourselves */ - s = gst_caps_get_structure (caps, 0); - gst_structure_get_int (s, "rate", &filter->samplerate); - gst_structure_get_int (s, "channels", &filter->channels); + gst_event_parse_caps (event, &caps); + ret = gst_my_filter_setcaps (filter, caps); break; } default: @@ -213,77 +280,122 @@ gst_my_filter_sink_event (GstPad *pad, } return ret; } -<!-- example-end forwardcaps.c b --> -<!-- example-begin forwardcaps.c c --><!-- -#include "chain.func" -#include "state.func" -#include "register.func" - --><!-- example-end forwardcaps.c c --></programlisting> + + [...] +]]> + </programlisting> + </sect2> + + <sect2 id="section-nego-dynamic"> + <title>Dynamic negotiation</title> + <para> + A last negotiation method is the most complex and powerful dynamic + negotiation. + </para> + <para> + Like with the transform negotiation in + <xref linkend="section-nego-transform"/>, dynamic negotiation will + perform a transformation on the downstream/upstream caps. Unlike the + transform negotiation, this transform will convert fixed caps to + unfixed caps. This means that the sink pad input caps can be converted + into unfixed (multiple) formats. The source pad will have to choose a + format from all the possibilities. It would usually like to choose a + format that requires the least amount of effort to produce but it does + not have to be. The selection of the format should also depend on the + caps that can be accepted downstream (see a QUERY_CAPS function in + <xref linkend="section-nego-getcaps"/>). + </para> + <para> + A typical flow goes like this: + </para> + <itemizedlist> + <listitem> + <para> + Caps are received on the sink pad of the element. + </para> + </listitem> + <listitem> + <para> + If the element prefers to operate in passthrough mode, check + if downstream accepts the caps with the ACCEPT_CAPS query. If it + does, we can complete negotiation and we can operate in + passthrough mode. + </para> + </listitem> + <listitem> + <para> + Calculate the possible caps for the source pad. + </para> + </listitem> + <listitem> + <para> + Query the downstream peer pad for the list of possible + caps. + </para> + </listitem> + <listitem> + <para> + Select from the downstream list the first caps that you can + transform to and set this as the output caps. You might have to + fixate the caps to some reasonable defaults to construct + fixed caps. + </para> + </listitem> + </itemizedlist> <para> - There may also be cases where the filter actually is able to - <emphasis>change</emphasis> the format of the stream. In those cases, - it will negotiate a new format. Obviously, the element should first - attempt to configure <quote>pass-through</quote>, which means that - it does not change the stream's format. However, if that fails, - then it should call <function>gst_pad_get_allowed_caps ()</function> - on its sourcepad to get a list of supported formats on the outputs, - and pick the first. The return value of that function is guaranteed - to be a subset of the template caps or NULL when there is no peer. + Examples of this type of elements include: </para> + <itemizedlist> + <listitem> + <para> + Converter elements such as videoconvert, audioconvert, audioresample, + videoscale, ... + </para> + </listitem> + <listitem> + <para> + Source elements such as audiotestsrc, videotestsrc, v4l2src, + pulsesrc, ... + </para> + </listitem> + </itemizedlist> <para> Let's look at the example of an element that can convert between samplerates, so where input and output samplerate don't have to be the same: </para> - <programlisting><!-- example-begin convertcaps.c a --><!-- -#include "init.func" -static GstCaps * -gst_my_filter_getcaps (GstPad * pad) -{ - return NULL; -} -static GstBuffer * -gst_my_filter_convert (GstMyFilter *filter, GstBuffer *in) -{ - return NULL; -} -static gboolean -gst_my_filter_event (GstPad * pad, GstEvent * event) -{ - return gst_pad_event_default (pad, event); -} ---><!-- example-end convertcaps.c a --> -<!-- example-begin convertcaps.c b --> + <programlisting> +<![CDATA[ static gboolean gst_my_filter_setcaps (GstMyFilter *filter, GstCaps *caps) { - if (gst_pad_set_caps (filter->sinkpad, caps)) { - filter->passthrough = TRUE; + if (gst_pad_set_caps (filter->sinkpad, caps)) { + filter->passthrough = TRUE; } else { GstCaps *othercaps, *newcaps; GstStructure *s = gst_caps_get_structure (caps, 0), *others; /* no passthrough, setup internal conversion */ - gst_structure_get_int (s, "channels", &filter->channels); - othercaps = gst_pad_get_allowed_caps (filter->srcpad); + gst_structure_get_int (s, "channels", &filter->channels); + othercaps = gst_pad_get_allowed_caps (filter->srcpad); others = gst_caps_get_structure (othercaps, 0); gst_structure_set (others, - "channels", G_TYPE_INT, filter->channels, NULL); + "channels", G_TYPE_INT, filter->channels, NULL); /* now, the samplerate value can optionally have multiple values, so * we "fixate" it, which means that one fixed value is chosen */ newcaps = gst_caps_copy_nth (othercaps, 0); gst_caps_unref (othercaps); - gst_pad_fixate_caps (filter->srcpad, newcaps); - if (!gst_pad_set_caps (filter->srcpad, newcaps)) + gst_pad_fixate_caps (filter->srcpad, newcaps); + if (!gst_pad_set_caps (filter->srcpad, newcaps)) return FALSE; /* we are now set up, configure internally */ - filter->passthrough = FALSE; - gst_structure_get_int (s, "rate", &filter->from_samplerate); + filter->passthrough = FALSE; + gst_structure_get_int (s, "rate", &filter->from_samplerate); others = gst_caps_get_structure (newcaps, 0); - gst_structure_get_int (others, "rate", &filter->to_samplerate); + gst_structure_get_int (others, "rate", &filter->to_samplerate); } return TRUE; @@ -302,7 +414,7 @@ gst_my_filter_sink_event (GstPad *pad, { GstCaps *caps; - gst_event_parse_caps (event, &caps); + gst_event_parse_caps (event, &caps); ret = gst_my_filter_setcaps (filter, caps); break; } @@ -322,44 +434,17 @@ gst_my_filter_chain (GstPad *pad, GstBuffer *out; /* push on if in passthrough mode */ - if (filter->passthrough) - return gst_pad_push (filter->srcpad, buf); + if (filter->passthrough) + return gst_pad_push (filter->srcpad, buf); /* convert, push */ out = gst_my_filter_convert (filter, buf); gst_buffer_unref (buf); - return gst_pad_push (filter->srcpad, out); + return gst_pad_push (filter->srcpad, out); } -<!-- example-end convertcaps.c b --> -<!-- example-begin convertcaps.c c --><!-- -#include "state.func" -#include "register.func" - --><!-- example-end convertcaps.c c --></programlisting> - </sect2> - - <sect2 id="section-nego-downstream-parse" - xreflabel="Parsing and setting caps"> - <title>Parsing and setting caps</title> - <para> - Other elements, such as certain types of decoders, will not be able - to parse the caps from their input, simply because the input format - does not contain the information required to know the output format - yet; rather, the data headers need to be parsed, too. In many cases, - fixed-caps will be enough, but in some cases, particularly in cases - where such decoders are renegotiable, it is also possible to use - full caps negotiation. - </para> - <para> - Fortunately, the code required to do so is very similar to the last - code example in <xref linkend="section-nego-downstream-embed"/>, with - the difference being that the caps is selected in the <function>_chain - ()</function>-function rather than in the <function>_event - ()</function>-function. The rest, as for getting all allowed caps from - the source pad, fixating and such, is all the same. Re-negotiation, - which will be handled in the next section, is very different for such - elements, though. - </para> +]]> + </programlisting> </sect2> </sect1> @@ -389,19 +474,40 @@ gst_my_filter_chain (GstPad *pad, <itemizedlist> <listitem> <para> - Elements that can be reconfigured on the srcpad should check its - NEED_RECONFIGURE flag with - <function>gst_pad_check_reconfigure ()</function> and it should - start renegotiation when the function returns TRUE. + Elements that want to propose a new format upstream need to first + check if the new caps are acceptable upstream with an ACCEPT_CAPS + query. Then they would send a RECONFIGURE event and be prepared to + answer the CAPS query with the new prefered format. It should be + noted that when there is no upstream element that can (or wants) + to renegotiate, the element needs to deal with the currently + configured format. + </para> + </listitem> + <listitem> + <para> + Elements that operate in transform negotiation according to + <xref linkend="section-nego-transform"/> pass the RECONFIGURE + event upstream. Because these elements simply do a fixed transform + based on the upstream caps, they need to send the event upstream + so that it can select a new format. + </para> + </listitem> + <listitem> + <para> + Elements that operate in fixed negotiation + (<xref linkend="section-nego-fixed"/>) drop the RECONFIGURE event. + These elements can't reconfigure and their output caps don't depend + on the upstream caps so the event can be dropped. </para> </listitem> <listitem> <para> - Elements that want to propose a new format upstream need to send - a RECONFIGURE event and be prepared to answer the CAPS query with - the new prefered format. It should be noted that when there is no - upstream element that can (or wants) to renegotiate, the element - needs to deal with the currently configured format. + Elements that can be reconfigured on the source pad (source pads + implementing dynamic negotiation in + <xref linkend="section-nego-dynamic"/>) should check its + NEED_RECONFIGURE flag with + <function>gst_pad_check_reconfigure ()</function> and it should + start renegotiation when the function returns TRUE. </para> </listitem> </itemizedlist> @@ -419,10 +525,8 @@ gst_my_filter_chain (GstPad *pad, </para> <para> </para> - <programlisting><!-- example-begin getcaps.c a --><!-- -#include "init.func" ---><!-- example-end getcaps.c a --> -<!-- example-begin getcaps.c b --> + <programlisting> +<![CDATA[ static gboolean gst_my_filter_query (GstPad *pad, GstObject * parent, GstQuery * query) { @@ -436,15 +540,15 @@ gst_my_filter_query (GstPad *pad, GstObject * parent, GstQuery * query) GstCaps *temp, *caps, *filter, *tcaps; gint i; - otherpad = (pad == filter->srcpad) ? filter->sinkpad : - filter->srcpad; + otherpad = (pad == filter->srcpad) ? filter->sinkpad : + filter->srcpad; caps = gst_pad_get_allowed_caps (otherpad); - gst_query_parse_caps (query, &filter); + gst_query_parse_caps (query, &filter); /* We support *any* samplerate, indifferent from the samplerate * supported by the linked elements on both sides. */ - for (i = 0; i < gst_caps_get_size (caps); i++) { + for (i = 0; i < gst_caps_get_size (caps); i++) { GstStructure *structure = gst_caps_get_structure (caps, i); gst_structure_remove_field (structure, "rate"); @@ -476,17 +580,17 @@ gst_my_filter_query (GstPad *pad, GstObject * parent, GstQuery * query) } return ret; } -<!-- example-end getcaps.c b --> -<!-- example-begin getcaps.c c --><!-- -static gboolean -gst_my_filter_setcaps (GstPad * pad, GstCaps * caps) -{ - return FALSE; -} -#include "chain.func" -#include "state.func" -#include "register.func" - --><!-- example-end getcaps.c c --></programlisting> +]]> + </programlisting> + </sect1> + + <sect1 id="section-nego-pullmode"> + <title>Pull-mode Caps negotiation</title> + <para> + WRITEME, the mechanism of pull-mode negotiation is not yet fully + understood. + </para> + <para> Using all the knowledge you've acquired by reading this chapter, you should be able to write an element that does correct caps negotiation. diff --git a/docs/pwg/advanced-qos.xml b/docs/pwg/advanced-qos.xml new file mode 100644 index 0000000..a6561bc --- /dev/null +++ b/docs/pwg/advanced-qos.xml @@ -0,0 +1,283 @@ +<chapter id="chapter-advanced-qos"> + <title>Quality Of Service (QoS)</title> + + <para> + Quality of Service in &GStreamer; is about measuring and ajusting + the real-time performance of a pipeline. The real-time performance is + always measured relative to the pipeline clock and typically happens in + the sinks when they synchronize buffers against the clock. + </para> + <para> + When buffers arrive late in the sink, i.e. when their running-time is + smaller than that of the clock, we say that the pipeline is having a + quality of service problem. These are a few possible reasons: + </para> + <itemizedlist> + <listitem> + <para> + High CPU load, there is not enough CPU power to handle the stream, + causing buffers to arrive late in the sink. + </para> + </listitem> + <listitem> + <para> + Network problems + </para> + </listitem> + <listitem> + <para> + Other resource problems such as disk load, memory bottlenecks etc + </para> + </listitem> + </itemizedlist> + <para> + The measurements result in QOS events that aim to adjust the datarate + in one or more upstream elements. Two types of adjustments can be + made: + </para> + <itemizedlist> + <listitem> + <para> + Short time "emergency" corrections based on latest observation in + the sinks. + </para> + <para> + Long term rate corrections based on trends observed in the sinks. + </para> + </listitem> + </itemizedlist> + <para> + It is also possible for the application to artificially introduce delay + between synchronized buffers, this is called throttling. It can be used + to limit or reduce the framerate, for example. + </para> + + <sect1 id="section-measuring"> + <title>Measuring QoS</title> + <para> + Elements that synchronize buffers on the pipeline clock will usually + measure the current QoS. They will also need to keep some statistics + in order to generate the QOS event. + </para> + <para> + For each buffer that arrives in the sink, the element needs to calculate + how late or how early it was. This is called the jitter. Negative jitter + values mean that the buffer was early, positive values mean that the + buffer was late. the jitter value gives an indication of how early/late + a buffer was. + </para> + <para> + A synchronizing element will also need to calculate how much time + elapsed between receiving two consecutive buffers. We call this the + processing time because that is the amount of time it takes for the + upstream element to produce/process the buffer. We can compare this + processing time to the duration of the buffer to have a measurement + of how fast upstream can produce data, called the proportion. + If, for example, upstream can produce a buffer in 0.5 seconds of 1 + second long, it is operating at twice the required speed. If, on the + other hand, it takes 2 seconds to produce a buffer with 1 seconds worth + of data, upstream is producing buffers too slow and we won't be able to + keep sycnhronization. Usually, a running average is kept of the + proportion. + </para> + <para> + A synchronizing element also needs to measure its own performance in + order to figure out if the performace problem is upstream of itself. + </para> + <para> + These measurements are used to construct a QOS event that is sent + upstream. Note that a QoS event is sent for each buffer that arrives + in the sink. + </para> + </sect1> + + <sect1 id="section-handling"> + <title>Handling QoS</title> + <para> + An element will have to install an event function on its source pads + in order to receive QOS events. Usually, the element will need to + store the value of the QOS event and use them in the data processing + function. The element will need to use a lock to protect these QoS + values as shown in the example below. Also make sure to pass the + QoS event upstream. + </para> + <programlisting> +<![CDATA[ + [...] + + case GST_EVENT_QOS: + { + GstQOSType type; + gdouble proportion; + GstClockTimeDiff diff; + GstClockTime timestamp; + + gst_event_parse_qos (event, &type, &proportion, &diff, ×tamp); + + GST_OBJECT_LOCK (decoder); + priv->qos_proportion = proportion; + priv->qos_timestamp = timestamp; + priv->qos_diff = diff; + GST_OBJECT_UNLOCK (decoder); + + res = gst_pad_push_event (decoder->sinkpad, event); + break; + } + + [...] +]]> + </programlisting> + <para> + With the QoS values, there are two types of corrections that an element + can do: + </para> + + <sect2 id="section-handling-short"> + <title>Short term correction</title> + <para> + The timestamp and the jitter value in the QOS event can be used to + perform a short term correction. If the jitter is positive, the + previous buffer arrived late and we can be sure that a buffer with + a timestamp < timestamp + jitter is also going to be late. We + can thus drop all buffers with a timestamp less than timestamp + + jitter. + </para> + <para> + If the buffer duration is known, a better estimation for the next + likely timestamp as: timestamp + 2 * jitter + duration. + </para> + <para> + A possible algorithm typically looks like this: + </para> + <programlisting> +<![CDATA[ + [...] + + GST_OBJECT_LOCK (dec); + qos_proportion = priv->qos_proportion; + qos_timestamp = priv->qos_timestamp; + qos_diff = priv->qos_diff; + GST_OBJECT_UNLOCK (dec); + + /* calculate the earliest valid timestamp */ + if (G_LIKELY (GST_CLOCK_TIME_IS_VALID (qos_timestamp))) { + if (G_UNLIKELY (qos_diff > 0)) { + earliest_time = qos_timestamp + 2 * qos_diff + frame_duration; + } else { + earliest_time = qos_timestamp + qos_diff; + } + } else { + earliest_time = GST_CLOCK_TIME_NONE; + } + + /* compare earliest_time to running-time of next buffer */ + if (earliest_time > timestamp) + goto drop_buffer; + + [...] +]]> + </programlisting> + </sect2> + + <sect2 id="section-handling-long"> + <title>Long term correction</title> + <para> + Long term corrections are a bit more difficult to perform. They + rely on the value of the propertion in the QOS event. Elements should + reduce the amount of resources they comsume by the proportion + field in the QoS message. + </para> + <para> + Here are some possible strategies to achieve this: + </para> + <itemizedlist> + <listitem> + <para> + Permanently dropping frames or reducing the CPU or bandwidth + requirements of the element. Some decoders might be able to + skip decoding of B frames. + </para> + </listitem> + <listitem> + <para> + Switch to lower quality processing or reduce the algorithmic + complexity. Care should be taken that this doesn't introduce + disturbing visual or audible glitches. + </para> + </listitem> + <listitem> + <para> + Switch to a lower quality source to reduce network bandwidth. + </para> + </listitem> + <listitem> + <para> + Assign more CPU cycles to critical parts of the pipeline. This + could, for example, be done by increasing the thread priority. + </para> + </listitem> + </itemizedlist> + <para> + In all cases, elements should be prepared to go back to their normal + processing rate when the proportion member in the QOS event approaches + the ideal proportion of 1.0 again. + </para> + </sect2> + </sect1> + + <sect1 id="section-throttle"> + <title>Throttling</title> + <para> + Elements synchronizing to the clock should expose a property to configure + them in throttle mode. In throttle mode, the time distance between buffers + is kept to a configurable throttle interval. This means that effectively + the buffer rate is limited to 1 buffer per throttle interval. This can be + used to limit the framerate, for example. + </para> + <para> + When an element is configured in throttling mode (this is usually only + implemented on sinks) it should produce QoS events upstream with the jitter + field set to the throttle interval. This should instruct upstream elements to + skip or drop the remaining buffers in the configured throttle interval. + </para> + <para> + The proportion field is set to the desired slowdown needed to get the + desired throttle interval. Implementations can use the QoS Throttle type, + the proportion and the jitter member to tune their implementations. + </para> + <para> + The default sink base class, has the <quote>throttle-time</quote> + property for this feature. You can test this with: + <command>gst-launch-1.0 videotestsrc ! + xvimagesink throttle-time=500000000</command> + </para> + </sect1> + + <sect1 id="section-messages"> + <title>QoS Messages</title> + <para> + In addition to the QOS events that are sent between elements in the + pipeline, there are also QOS messages posted on the pipeline bus to + inform the application of QoS decisions. The QOS message contains + the timestamps of when something was dropped along with the amount + of dropped vs processed items. Elements must post a QOS + message under these conditions: + </para> + <itemizedlist> + <listitem> + <para> + The element dropped a buffer because of QoS reasons. + </para> + </listitem> + <listitem> + <para> + An element changes its processing strategy because of QoS reasons + (quality). This could include a decoder that decides to drop every + B frame to increase its processing speed or an effect element + switching to a lower quality algorithm. + </para> + </listitem> + </itemizedlist> + </sect1> + +</chapter> diff --git a/docs/pwg/building-boiler.xml b/docs/pwg/building-boiler.xml index 7cba279..77d78ce 100644 --- a/docs/pwg/building-boiler.xml +++ b/docs/pwg/building-boiler.xml @@ -378,6 +378,7 @@ gst_my_filter_init (GstMyFilter * filter) correct template would look like this: </para> <programlisting> +<![CDATA[ static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ( "sink", @@ -390,6 +391,7 @@ GST_STATIC_PAD_TEMPLATE ( "rate = (int) [ 8000, 96000 ]" ) ); +]]> </programlisting> <para> Values surrounded by curly brackets (<quote>{</quote> and @@ -428,7 +430,9 @@ GST_STATIC_PAD_TEMPLATE ( Also, in this function, any supported element type in the plugin should be registered. </para> - <programlisting><!-- example-begin register.func --> + <programlisting> +<!-- example-begin register.func --> +<![CDATA[ static gboolean plugin_init (GstPlugin *plugin) { @@ -448,7 +452,9 @@ GST_PLUGIN_DEFINE ( "GStreamer", "http://gstreamer.net/" ) - <!-- example-end register.func --></programlisting> +]]> +<!-- example-end register.func --> + </programlisting> <para> Note that the information returned by the plugin_init() function will be cached in a central registry. For this reason, it is important that the diff --git a/docs/pwg/building-eventfn.xml b/docs/pwg/building-eventfn.xml index 51836ef..b72e376 100644 --- a/docs/pwg/building-eventfn.xml +++ b/docs/pwg/building-eventfn.xml @@ -13,18 +13,11 @@ Below follows a very simple event function that we install on the sink pad of our element. </para> - <programlisting><!-- example-begin event.c a --><!-- -#include "init.func" -#include "caps.func" -static gboolean -gst_my_filter_event (GstPad * pad, GstObject * parent, GstEvent * event) -{ - return gst_pad_event_default (pad, parent, event); -} ---><!-- example-end event.c a --> + <programlisting> +<![CDATA[ static gboolean gst_my_filter_sink_event (GstPad *pad, GstObject *parent, - GstBuffer *buf); + GstEvent *event); [..] @@ -34,7 +27,7 @@ gst_my_filter_init (GstMyFilter * filter) [..] /* configure event function on the pad before adding * the pad to the element */ - gst_pad_set_event_function (filter->sinkpad, + gst_pad_set_event_function (filter->sinkpad, gst_my_filter_sink_event); [..] } @@ -67,6 +60,7 @@ gst_my_filter_sink_event (GstPad *pad, } return ret; } +]]> </programlisting> <para> It is a good idea to call the default event handler diff --git a/docs/pwg/building-queryfn.xml b/docs/pwg/building-queryfn.xml new file mode 100644 index 0000000..ad2e943 --- /dev/null +++ b/docs/pwg/building-queryfn.xml @@ -0,0 +1,72 @@ + +<!-- ############ chapter ############# --> + +<chapter id="chapter-building-queryfn"> + <title>The query function</title> + <para> + Through the query function, your element will receive queries that it + has to reply to. These are queries like position, duration but also + about the supported formats and scheduling modes your element supports. + Queries can travel both upstream and downstream, so you can receive them + on sink pads as well as source pads. + </para> + <para> + Below follows a very simple query function that we install on the source + pad of our element. + </para> + <programlisting> +<![CDATA[ +static gboolean gst_my_filter_src_query (GstPad *pad, + GstObject *parent, + GstQuery *query); + +[..] + +static void +gst_my_filter_init (GstMyFilter * filter) +{ +[..] + /* configure event function on the pad before adding + * the pad to the element */ + gst_pad_set_event_function (filter->srcpad, + gst_my_filter_src_event); +[..] +} + +static gboolean +gst_my_filter_src_query (GstPad *pad, + GstObject *parent, + GstQuery *query) +{ + gboolean ret; + GstMyFilter *filter = GST_MY_FILTER (parent); + + switch (GST_QUERY_TYPE (query)) { + case GST_QUERY_POSITION: + /* we should report the current position */ + [...] + break; + case GST_QUERY_DURATION: + /* we should report the duration here */ + [...] + break; + case GST_QUERY_CAPS: + /* we should report the supported caps here */ + [...] + break; + default: + /* just call the default handler */ + ret = gst_pad_query_default (pad, parent, query); + break; + } + return ret; +} +]]> + </programlisting> + <para> + It is a good idea to call the default query handler + <function>gst_pad_query_default ()</function> for unknown queries. + Depending on the query type, the default handler will forward + the query or simply unref it. + </para> +</chapter> diff --git a/docs/pwg/intro-basics.xml b/docs/pwg/intro-basics.xml index 4a50e91..e864e5e 100644 --- a/docs/pwg/intro-basics.xml +++ b/docs/pwg/intro-basics.xml @@ -220,6 +220,7 @@ GstAllocator. Elements can ask a GstBufferPool or GstAllocator from the downstream peer element. If downstream is able to provide these objects, upstream can use them to allocate buffers. + See more in <xref linkend="chapter-allocation"/>. </para> <para> Many sink elements have accelerated methods for copying data @@ -254,7 +255,8 @@ for ensuring that the parameters required to fully specify a format match up correctly when linking pads between elements. Each link that is made between elements has a specified type and optionally a set of - properties. + properties. See more about caps negotiation in + <xref linkend="chapter-negotiation"/>. </para> <!-- ############ sect2 ############# --> diff --git a/docs/pwg/pwg.xml b/docs/pwg/pwg.xml index 6d91f26..4f79709 100644 --- a/docs/pwg/pwg.xml +++ b/docs/pwg/pwg.xml @@ -18,22 +18,24 @@ <!ENTITY BUILDING_PADS SYSTEM "building-pads.xml"> <!ENTITY BUILDING_CHAINFN SYSTEM "building-chainfn.xml"> <!ENTITY BUILDING_EVENTFN SYSTEM "building-eventfn.xml"> +<!ENTITY BUILDING_QUERYFN SYSTEM "building-queryfn.xml"> <!ENTITY BUILDING_STATE SYSTEM "building-state.xml"> <!ENTITY BUILDING_PROPS SYSTEM "building-props.xml"> <!ENTITY BUILDING_SIGNALS SYSTEM "building-signals.xml"> <!ENTITY BUILDING_TESTAPP SYSTEM "building-testapp.xml"> <!-- Part 3: Advanced Filter Concepts --> +<!ENTITY ADVANCED_REQUEST SYSTEM "advanced-request.xml"> +<!ENTITY ADVANCED_SCHEDULING SYSTEM "advanced-scheduling.xml"> <!ENTITY ADVANCED_NEGOTIATION SYSTEM "advanced-negotiation.xml"> <!ENTITY ADVANCED_ALLOCATION SYSTEM "advanced-allocation.xml"> -<!ENTITY ADVANCED_SCHEDULING SYSTEM "advanced-scheduling.xml"> <!ENTITY ADVANCED_TYPES SYSTEM "advanced-types.xml"> -<!ENTITY ADVANCED_REQUEST SYSTEM "advanced-request.xml"> +<!ENTITY ADVANCED_EVENTS SYSTEM "advanced-events.xml"> <!ENTITY ADVANCED_CLOCK SYSTEM "advanced-clock.xml"> +<!ENTITY ADVANCED_QOS SYSTEM "advanced-qos.xml"> <!ENTITY ADVANCED_DPARAMS SYSTEM "advanced-dparams.xml"> <!ENTITY ADVANCED_INTERFACES SYSTEM "advanced-interfaces.xml"> <!ENTITY ADVANCED_TAGGING SYSTEM "advanced-tagging.xml"> -<!ENTITY ADVANCED_EVENTS SYSTEM "advanced-events.xml"> <!-- Part 4: Creating special element types --> <!ENTITY OTHER_BASE SYSTEM "other-base.xml"> @@ -119,6 +121,7 @@ &BUILDING_PADS; &BUILDING_CHAINFN; &BUILDING_EVENTFN; + &BUILDING_QUERYFN; &BUILDING_STATE; &BUILDING_PROPS; &BUILDING_SIGNALS; @@ -140,16 +143,17 @@ </para> </partintro> + &ADVANCED_REQUEST; + &ADVANCED_SCHEDULING; &ADVANCED_NEGOTIATION; &ADVANCED_ALLOCATION; - &ADVANCED_SCHEDULING; &ADVANCED_TYPES; - &ADVANCED_REQUEST; + &ADVANCED_EVENTS; &ADVANCED_CLOCK; + &ADVANCED_QOS; &ADVANCED_DPARAMS; &ADVANCED_INTERFACES; &ADVANCED_TAGGING; - &ADVANCED_EVENTS; <!-- FIXME: add querying, event handling and conversion --> |