misc utils

Also this include:
* Mp4 DASH reader/writter
* WebM reader/writter
* a subtitle converter for Timed Text Markup Language v1 and TranScript (v1, v2 and v3)
* SharpStream to wrap IntputStream and OutputStream in one interface
* custom implementation of DataInputStream

app/src/main/java/org/schabi/newpipe/streams/DataReader.java

@@ -0,0 +1,103 @@
+package org.schabi.newpipe.streams;
+
+import java.io.EOFException;
+import java.io.IOException;
+
+import org.schabi.newpipe.streams.io.SharpStream;
+
+/**
+ * @author kapodamy
+ */
+public class DataReader {
+
+    public final static int SHORT_SIZE = 2;
+    public final static int LONG_SIZE = 8;
+    public final static int INTEGER_SIZE = 4;
+    public final static int FLOAT_SIZE = 4;
+
+    private long pos;
+    public final SharpStream stream;
+    private final boolean rewind;
+
+    public DataReader(SharpStream stream) {
+        this.rewind = stream.canRewind();
+        this.stream = stream;
+        this.pos = 0L;
+    }
+
+    public long position() {
+        return pos;
+    }
+
+    public final int readInt() throws IOException {
+        primitiveRead(INTEGER_SIZE);
+        return primitive[0] << 24 | primitive[1] << 16 | primitive[2] << 8 | primitive[3];
+    }
+
+    public final int read() throws IOException {
+        int value = stream.read();
+        if (value == -1) {
+            throw new EOFException();
+        }
+
+        pos++;
+        return value;
+    }
+
+    public final long skipBytes(long amount) throws IOException {
+        amount = stream.skip(amount);
+        pos += amount;
+        return amount;
+    }
+
+    public final long readLong() throws IOException {
+        primitiveRead(LONG_SIZE);
+        long high = primitive[0] << 24 | primitive[1] << 16 | primitive[2] << 8 | primitive[3];
+        long low = primitive[4] << 24 | primitive[5] << 16 | primitive[6] << 8 | primitive[7];
+        return high << 32 | low;
+    }
+
+    public final short readShort() throws IOException {
+        primitiveRead(SHORT_SIZE);
+        return (short) (primitive[0] << 8 | primitive[1]);
+    }
+
+    public final int read(byte[] buffer) throws IOException {
+        return read(buffer, 0, buffer.length);
+    }
+
+    public final int read(byte[] buffer, int offset, int count) throws IOException {
+        int res = stream.read(buffer, offset, count);
+        pos += res;
+
+        return res;
+    }
+
+    public final boolean available() {
+        return stream.available() > 0;
+    }
+
+    public void rewind() throws IOException {
+        stream.rewind();
+        pos = 0;
+    }
+
+    public boolean canRewind() {
+        return rewind;
+    }
+
+    private short[] primitive = new short[LONG_SIZE];
+
+    private void primitiveRead(int amount) throws IOException {
+        byte[] buffer = new byte[amount];
+        int read = stream.read(buffer, 0, amount);
+        pos += read;
+        if (read != amount) {
+            throw new EOFException("Truncated data, missing " + String.valueOf(amount - read) + " bytes");
+        }
+
+        for (int i = 0; i < buffer.length; i++) {
+            primitive[i] = (short) (buffer[i] & 0xFF);// the "byte" datatype is signed and is very annoying
+        }
+    }
+}

app/src/main/java/org/schabi/newpipe/streams/SubtitleConverter.java

@@ -0,0 +1,370 @@
+package org.schabi.newpipe.streams;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.xml.sax.SAXException;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.text.ParseException;
+import java.util.Locale;
+
+import org.schabi.newpipe.streams.io.SharpStream;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.xpath.XPathExpressionException;
+
+/**
+ * @author kapodamy
+ */
+public class SubtitleConverter {
+    private static final String NEW_LINE = "\r\n";
+
+    public void dumpTTML(SharpStream in, final SharpStream out, final boolean ignoreEmptyFrames, final boolean detectYoutubeDuplicateLines
+    ) throws IOException, ParseException, SAXException, ParserConfigurationException, XPathExpressionException {
+        
+        final FrameWriter callback = new FrameWriter() {
+            int frameIndex = 0;
+            final Charset charset = Charset.forName("utf-8");
+            
+            @Override
+            public void yield(SubtitleFrame frame) throws IOException {
+                if (ignoreEmptyFrames && frame.isEmptyText()) {
+                    return;
+                }
+                out.write(String.valueOf(frameIndex++).getBytes(charset));
+                out.write(NEW_LINE.getBytes(charset));
+                out.write(getTime(frame.start, true).getBytes(charset));
+                out.write(" --> ".getBytes(charset));
+                out.write(getTime(frame.end, true).getBytes(charset));
+                out.write(NEW_LINE.getBytes(charset));
+                out.write(frame.text.getBytes(charset));
+                out.write(NEW_LINE.getBytes(charset));
+                out.write(NEW_LINE.getBytes(charset));
+            }
+        };
+        
+        read_xml_based(in, callback, detectYoutubeDuplicateLines,
+                "tt", "xmlns", "http://www.w3.org/ns/ttml",
+                new String[]{"timedtext", "head", "wp"},
+                new String[]{"body", "div", "p"},
+                "begin", "end", true
+        );                  
+    }
+
+    private void read_xml_based(SharpStream source, FrameWriter callback, boolean detectYoutubeDuplicateLines,
+                                String root, String formatAttr, String formatVersion, String[] cuePath, String[] framePath,
+                                String timeAttr, String durationAttr, boolean hasTimestamp
+    ) throws IOException, ParseException, SAXException, ParserConfigurationException, XPathExpressionException {
+        /*
+         * XML based subtitles parser with BASIC support
+         * multiple CUE is not supported
+         * styling is not supported
+         * tag timestamps (in auto-generated subtitles) are not supported, maybe in the future
+         * also TimestampTagOption enum is not applicable
+         * Language parsing is not supported
+         */
+
+        byte[] buffer = new byte[source.available()];
+        source.read(buffer);
+
+        DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+        factory.setNamespaceAware(true);
+        DocumentBuilder builder = factory.newDocumentBuilder();
+        Document xml = builder.parse(new ByteArrayInputStream(buffer));
+
+        String attr;
+
+        // get the format version or namespace
+        Element node = xml.getDocumentElement();
+
+        if (node == null) {
+            throw new ParseException("Can't get the format version. ¿wrong namespace?", -1);
+        } else if (!node.getNodeName().equals(root)) {
+            throw new ParseException("Invalid root", -1);
+        }
+
+        if (formatAttr.equals("xmlns")) {
+            if (!node.getNamespaceURI().equals(formatVersion)) {
+                throw new UnsupportedOperationException("Expected xml namespace: " + formatVersion);
+            }
+        } else {
+            attr = node.getAttributeNS(formatVersion, formatAttr);
+            if (attr == null) {
+                throw new ParseException("Can't get the format attribute", -1);
+            }
+            if (!attr.equals(formatVersion)) {
+                throw new ParseException("Invalid format version : " + attr, -1);
+            }
+        }
+
+        NodeList node_list;
+
+        int line_break = 0;// Maximum characters per line if present (valid for TranScript v3)
+
+        if (!hasTimestamp) {
+            node_list = selectNodes(xml, cuePath, formatVersion);
+
+            if (node_list != null) {
+                // if the subtitle has multiple CUEs, use the highest value
+                for (int i = 0; i < node_list.getLength(); i++) {
+                    try {
+                        int tmp = Integer.parseInt(((Element) node_list.item(i)).getAttributeNS(formatVersion, "ah"));
+                        if (tmp > line_break) {
+                            line_break = tmp;
+                        }
+                    } catch (Exception err) {
+                    }
+                }
+            }
+        }
+
+        // parse every frame
+        node_list = selectNodes(xml, framePath, formatVersion);
+
+        if (node_list == null) {
+            return;// no frames detected
+        }
+
+        int fs_ff = -1;// first timestamp of first frame
+        boolean limit_lines = false;
+
+        for (int i = 0; i < node_list.getLength(); i++) {
+            Element elem = (Element) node_list.item(i);
+            SubtitleFrame obj = new SubtitleFrame();
+            obj.text = elem.getTextContent();
+
+            attr = elem.getAttribute(timeAttr);// ¡this cant be null!
+            obj.start = hasTimestamp ? parseTimestamp(attr) : Integer.parseInt(attr);
+
+            attr = elem.getAttribute(durationAttr);
+            if (obj.text == null || attr == null) {
+                continue;// normally is a blank line (on auto-generated subtitles) ignore
+            }
+
+            if (hasTimestamp) {
+                obj.end = parseTimestamp(attr);
+
+                if (detectYoutubeDuplicateLines) {
+                    if (limit_lines) {
+                        int swap = obj.end;
+                        obj.end = fs_ff;
+                        fs_ff = swap;
+                    } else {
+                        if (fs_ff < 0) {
+                            fs_ff = obj.end;
+                        } else {
+                            if (fs_ff < obj.start) {
+                                limit_lines = true;// the subtitles has duplicated lines
+                            } else {
+                                detectYoutubeDuplicateLines = false;
+                            }
+                        }
+                    }
+                }
+            } else {
+                obj.end = obj.start + Integer.parseInt(attr);
+            }
+
+            if (/*node.getAttribute("w").equals("1") &&*/line_break > 1 && obj.text.length() > line_break) {
+
+                // implement auto line breaking (once)
+                StringBuilder text = new StringBuilder(obj.text);
+                obj.text = null;
+
+                switch (text.charAt(line_break)) {
+                    case ' ':
+                    case '\t':
+                        putBreakAt(line_break, text);
+                        break;
+                    default:// find the word start position
+                        for (int j = line_break - 1; j > 0; j--) {
+                            switch (text.charAt(j)) {
+                                case ' ':
+                                case '\t':
+                                    putBreakAt(j, text);
+                                    j = -1;
+                                    break;
+                                case '\r':
+                                case '\n':
+                                    j = -1;// long word, just ignore
+                                    break;
+                            }
+                        }
+                        break;
+                }
+
+                obj.text = text.toString();// set the processed text
+            }
+
+            callback.yield(obj);
+        }
+    }
+
+    private static NodeList selectNodes(Document xml, String[] path, String namespaceUri) throws XPathExpressionException {
+        Element ref = xml.getDocumentElement();
+
+        for (int i = 0; i < path.length - 1; i++) {
+            NodeList nodes = ref.getChildNodes();
+            if (nodes.getLength() < 1) {
+                return null;
+            }
+
+            Element elem;
+            for (int j = 0; j < nodes.getLength(); j++) {
+                if (nodes.item(j).getNodeType() == Node.ELEMENT_NODE) {
+                    elem = (Element) nodes.item(j);
+                    if (elem.getNodeName().equals(path[i]) && elem.getNamespaceURI().equals(namespaceUri)) {
+                        ref = elem;
+                        break;
+                    }
+                }
+            }
+        }
+
+        return ref.getElementsByTagNameNS(namespaceUri, path[path.length - 1]);
+    }
+
+    private static int parseTimestamp(String multiImpl) throws NumberFormatException, ParseException {
+        if (multiImpl.length() < 1) {
+            return 0;
+        } else if (multiImpl.length() == 1) {
+            return Integer.parseInt(multiImpl) * 1000;// ¡this must be a number in seconds!
+        }
+
+        // detect wallclock-time
+        if (multiImpl.startsWith("wallclock(")) {
+            throw new UnsupportedOperationException("Parsing wallclock timestamp is not implemented");
+        }
+
+        // detect offset-time
+        if (multiImpl.indexOf(':') < 0) {
+            int multiplier = 1000;
+            char metric = multiImpl.charAt(multiImpl.length() - 1);
+            switch (metric) {
+                case 'h':
+                    multiplier *= 3600000;
+                    break;
+                case 'm':
+                    multiplier *= 60000;
+                    break;
+                case 's':
+                    if (multiImpl.charAt(multiImpl.length() - 2) == 'm') {
+                        multiplier = 1;// ms
+                    }
+                    break;
+                default:
+                    if (!Character.isDigit(metric)) {
+                        throw new NumberFormatException("Invalid metric suffix found on : " + multiImpl);
+                    }
+                    metric = '\0';
+                    break;
+            }
+            try {
+                String offset_time = multiImpl;
+
+                if (multiplier == 1) {
+                    offset_time = offset_time.substring(0, offset_time.length() - 2);
+                } else if (metric != '\0') {
+                    offset_time = offset_time.substring(0, offset_time.length() - 1);
+                }
+
+                double time_metric_based = Double.parseDouble(offset_time);
+                if (Math.abs(time_metric_based) <= Double.MAX_VALUE) {
+                    return (int) (time_metric_based * multiplier);
+                }
+            } catch (Exception err) {
+                throw new UnsupportedOperationException("Invalid or not implemented timestamp on: " + multiImpl);
+            }
+        }
+
+        // detect clock-time
+        int time = 0;
+        String[] units = multiImpl.split(":");
+
+        if (units.length < 3) {
+            throw new ParseException("Invalid clock-time timestamp", -1);
+        }
+
+        time += Integer.parseInt(units[0]) * 3600000;// hours
+        time += Integer.parseInt(units[1]) * 60000;//minutes
+        time += Float.parseFloat(units[2]) * 1000f;// seconds and milliseconds (if present)
+
+        // frames and sub-frames are ignored (not implemented)
+        // time += units[3] * fps;
+        return time;
+    }
+
+    private static void putBreakAt(int idx, StringBuilder str) {
+        // this should be optimized at compile time
+
+        if (NEW_LINE.length() > 1) {
+            str.delete(idx, idx + 1);// remove after replace
+            str.insert(idx, NEW_LINE);
+        } else {
+            str.setCharAt(idx, NEW_LINE.charAt(0));
+        }
+    }
+
+    private static String getTime(int time, boolean comma) {
+        // cast every value to integer to avoid auto-round in ToString("00").
+        StringBuilder str = new StringBuilder(12);
+        str.append(numberToString(time / 1000 / 3600, 2));// hours
+        str.append(':');
+        str.append(numberToString(time / 1000 / 60 % 60, 2));// minutes
+        str.append(':');
+        str.append(numberToString(time / 1000 % 60, 2));// seconds
+        str.append(comma ? ',' : '.');
+        str.append(numberToString(time % 1000, 3));// miliseconds
+
+        return str.toString();
+    }
+
+    private static String numberToString(int nro, int pad) {
+        return String.format(Locale.ENGLISH, "%0".concat(String.valueOf(pad)).concat("d"), nro);
+    }
+
+
+    /******************
+     * helper classes *
+     ******************/
+
+    private interface FrameWriter {
+
+        void yield(SubtitleFrame frame) throws IOException;
+    }
+
+    private static class SubtitleFrame {
+        //Java no support unsigned int
+
+        public int end;
+        public int start;
+        public String text = "";
+
+        private boolean isEmptyText() {
+            if (text == null) {
+                return true;
+            }
+
+            for (int i = 0; i < text.length(); i++) {
+                switch (text.charAt(i)) {
+                    case ' ':
+                    case '\t':
+                    case '\r':
+                    case '\n':
+                        break;
+                    default:
+                        return false;
+                }
+            }
+
+            return true;
+        }
+    }
+
+}

app/src/main/java/org/schabi/newpipe/streams/TrackDataChunk.java

@@ -0,0 +1,65 @@
+package org.schabi.newpipe.streams;
+
+import java.io.InputStream;
+import java.io.IOException;
+
+public class TrackDataChunk extends InputStream {
+
+    private final DataReader base;
+    private int size;
+
+    public TrackDataChunk(DataReader base, int size) {
+        this.base = base;
+        this.size = size;
+    }
+
+    @Override
+    public int read() throws IOException {
+        if (size < 1) {
+            return -1;
+        }
+
+        int res = base.read();
+
+        if (res >= 0) {
+            size--;
+        }
+
+        return res;
+    }
+
+    @Override
+    public int read(byte[] buffer) throws IOException {
+        return read(buffer, 0, buffer.length);
+    }
+
+    @Override
+    public int read(byte[] buffer, int offset, int count) throws IOException {
+        count = Math.min(size, count);
+        int read = base.read(buffer, offset, count);
+        size -= count;
+        return read;
+    }
+
+    @Override
+    public long skip(long amount) throws IOException {
+        long res = base.skipBytes(Math.min(amount, size));
+        size -= res;
+        return res;
+    }
+
+    @Override
+    public int available() {
+        return size;
+    }
+
+    @Override
+    public void close() {
+        size = 0;
+    }
+
+    @Override
+    public boolean markSupported() {
+        return false;
+    }
+}

app/src/main/java/org/schabi/newpipe/streams/io/SharpStream.java

@@ -0,0 +1,47 @@
+package org.schabi.newpipe.streams.io;
+
+import java.io.IOException;
+
+/**
+ * based c#
+ */
+public abstract class SharpStream {
+
+    public abstract int read() throws IOException;
+
+    public abstract int read(byte buffer[]) throws IOException;
+
+    public abstract int read(byte buffer[], int offset, int count) throws IOException;
+
+    public abstract long skip(long amount) throws IOException;
+
+
+    public abstract int available();
+
+    public abstract void rewind() throws IOException;
+
+
+    public abstract void dispose();
+
+    public abstract boolean isDisposed();
+
+
+    public abstract boolean canRewind();
+
+    public abstract boolean canRead();
+
+    public abstract boolean canWrite();
+
+
+    public abstract void write(byte value) throws IOException;
+
+    public abstract void write(byte[] buffer) throws IOException;
+
+    public abstract void write(byte[] buffer, int offset, int count) throws IOException;
+
+    public abstract void flush() throws IOException;
+
+    public void setLength(long length) throws IOException {
+        throw new IOException("Not implemented");
+    }
+}

app/src/main/java/us/shandian/giga/postprocessing/Mp4DashMuxer.java

@@ -1,7 +1,7 @@
 package us.shandian.giga.postprocessing;
 
-import org.schabi.newpipe.extractor.utils.Mp4DashWriter;
-import org.schabi.newpipe.extractor.utils.io.SharpStream;
+import org.schabi.newpipe.streams.Mp4DashWriter;
+import org.schabi.newpipe.streams.io.SharpStream;
 
 import java.io.IOException;
 

app/src/main/java/us/shandian/giga/postprocessing/Postprocessing.java

@@ -2,7 +2,7 @@ package us.shandian.giga.postprocessing;
 
 import android.os.Message;
 
-import org.schabi.newpipe.extractor.utils.io.SharpStream;
+import org.schabi.newpipe.streams.io.SharpStream;
 
 import java.io.File;
 import java.io.IOException;

app/src/main/java/us/shandian/giga/postprocessing/TestAlgo.java

@@ -2,7 +2,7 @@ package us.shandian.giga.postprocessing;
 
 import android.util.Log;
 
-import org.schabi.newpipe.extractor.utils.io.SharpStream;
+import org.schabi.newpipe.streams.io.SharpStream;
 
 import java.io.IOException;
 import java.util.Random;

app/src/main/java/us/shandian/giga/postprocessing/TttmlConverter.java

@@ -1,16 +1,25 @@
 package us.shandian.giga.postprocessing;
 
-import org.schabi.newpipe.extractor.utils.io.SharpStream;
-import org.schabi.newpipe.extractor.utils.SubtitleConverter;
+import android.util.Log;
+
+import org.schabi.newpipe.streams.io.SharpStream;
+import org.schabi.newpipe.streams.SubtitleConverter;
+import org.xml.sax.SAXException;
 
 import java.io.IOException;
+import java.text.ParseException;
+
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.xpath.XPathExpressionException;
 
 import us.shandian.giga.get.DownloadMission;
 import us.shandian.giga.postprocessing.io.SharpInputStream;
+
 /**
  * @author kapodamy
  */
 class TttmlConverter extends Postprocessing {
+    private static final String TAG = "TttmlConverter"; 
 
     TttmlConverter(DownloadMission mission) {
         super(mission);
@@ -26,14 +35,32 @@ class TttmlConverter extends Postprocessing {
         if (format == null || format.equals("ttml")) {
             SubtitleConverter ttmlDumper = new SubtitleConverter();
 
-            int res = ttmlDumper.dumpTTML(
-                    sources[0],
-                    out,
-                    getArgumentAt(1, "true").equals("true"),
-                    getArgumentAt(2, "true").equals("true")
-            );
+            try {
+                ttmlDumper.dumpTTML(
+                        sources[0],
+                        out,
+                        getArgumentAt(1, "true").equals("true"),
+                        getArgumentAt(2, "true").equals("true")
+                );   
+            } catch (Exception err) {
+                Log.e(TAG, "subtitle parse failed", err);
 
-            return res == 0 ? OK_RESULT : res;
+                if (err instanceof IOException) {
+                    return 1;
+                } else if (err instanceof ParseException) {
+                    return 2;
+                } else if (err instanceof SAXException) {
+                    return 3;
+                } else if (err instanceof ParserConfigurationException) {
+                    return 4;
+                } else if (err instanceof XPathExpressionException) {
+                    return 7;
+                }
+                
+                return 8;
+            }
+
+            return OK_RESULT;
         } else if (format.equals("srt")) {
             byte[] buffer = new byte[8 * 1024];
             int read;

app/src/main/java/us/shandian/giga/postprocessing/WebMMuxer.java

@@ -1,9 +1,9 @@
 package us.shandian.giga.postprocessing;
 
-import org.schabi.newpipe.extractor.utils.WebMReader.TrackKind;
-import org.schabi.newpipe.extractor.utils.WebMReader.WebMTrack;
-import org.schabi.newpipe.extractor.utils.WebMWriter;
-import org.schabi.newpipe.extractor.utils.io.SharpStream;
+import org.schabi.newpipe.streams.WebMReader.TrackKind;
+import org.schabi.newpipe.streams.WebMReader.WebMTrack;
+import org.schabi.newpipe.streams.WebMWriter;
+import org.schabi.newpipe.streams.io.SharpStream;
 
 import java.io.IOException;
 

app/src/main/java/us/shandian/giga/postprocessing/io/ChunkFileInputStream.java

@@ -1,6 +1,6 @@
 package us.shandian.giga.postprocessing.io;
 
-import org.schabi.newpipe.extractor.utils.io.SharpStream;
+import org.schabi.newpipe.streams.io.SharpStream;
 
 import java.io.File;
 import java.io.IOException;

app/src/main/java/us/shandian/giga/postprocessing/io/CircularFile.java

@@ -1,6 +1,6 @@
 package us.shandian.giga.postprocessing.io;
 
-import org.schabi.newpipe.extractor.utils.io.SharpStream;
+import org.schabi.newpipe.streams.io.SharpStream;
 
 import java.io.File;
 import java.io.IOException;

app/src/main/java/us/shandian/giga/postprocessing/io/FileStream.java

@@ -1,6 +1,6 @@
 package us.shandian.giga.postprocessing.io;
 
-import org.schabi.newpipe.extractor.utils.io.SharpStream;
+import org.schabi.newpipe.streams.io.SharpStream;
 
 import java.io.IOException;
 import java.io.RandomAccessFile;

app/src/main/java/us/shandian/giga/postprocessing/io/SharpInputStream.java

@@ -7,7 +7,7 @@ package us.shandian.giga.postprocessing.io;
 
 import android.support.annotation.NonNull;
 
-import org.schabi.newpipe.extractor.utils.io.SharpStream;
+import org.schabi.newpipe.streams.io.SharpStream;
 
 import java.io.IOException;
 import java.io.InputStream;