neoris/NewPipe
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

use java error system in the crawler

Browse Source
This commit is contained in:
Christian Schabesberger 2016-01-31 19:57:30 +01:00
parent 46c2db310a
commit 7f12b58722
17 changed files with 877 additions and 673 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
app/src/androidTest/java/org/schabi/newpipe/services/youtube/YoutubeSearchEngineTest.java
View File

@ -3,7 +3,9 @@ package org.schabi.newpipe.services.youtube;
import android.test.AndroidTestCase;
import org.schabi.newpipe.crawler.VideoPreviewInfo;
import org.schabi.newpipe.services.SearchEngine;
import org.schabi.newpipe.crawler.SearchEngine;
import org.schabi.newpipe.crawler.services.youtube.YoutubeSearchEngine;
import org.schabi.newpipe.Downloader;
import java.util.ArrayList;
@ -35,8 +37,9 @@ public class YoutubeSearchEngineTest extends AndroidTestCase {
public void setUp() throws Exception{
super.setUp();
SearchEngine engine = new YoutubeSearchEngine();
result = engine.search("https://www.youtube.com/results?search_query=bla", 0, "de");
suggestionReply = engine.suggestionList("hello");
result = engine.search("https://www.youtube.com/results?search_query=bla",
0, "de", new Downloader());
suggestionReply = engine.suggestionList("hello", new Downloader());
}
public void testIfNoErrorOccur() {

53
app/src/androidTest/java/org/schabi/newpipe/services/youtube/YoutubeVideoExtractorDefaultTest.java
View File

@ -1,9 +1,14 @@
package org.schabi.newpipe.services.youtube;
import android.test.AndroidTestCase;
import android.util.Log;
import org.schabi.newpipe.services.VideoInfo;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.crawler.CrawlingException;
import org.schabi.newpipe.crawler.ParsingException;
import org.schabi.newpipe.crawler.services.youtube.YoutubeVideoExtractor;
import org.schabi.newpipe.crawler.VideoInfo;
import java.io.IOException;
/**
* Created by the-scrabi on 30.12.15.
@ -28,58 +33,58 @@ import org.schabi.newpipe.services.VideoInfo;
public class YoutubeVideoExtractorDefaultTest extends AndroidTestCase {
private YoutubeVideoExtractor extractor;
public void setUp() {
extractor = new YoutubeVideoExtractor("https://www.youtube.com/watch?v=FmG385_uUys");
public void setUp() throws IOException, CrawlingException {
extractor = new YoutubeVideoExtractor("https://www.youtube.com/watch?v=FmG385_uUys",
new Downloader());
}
public void testGetErrorCode() {
assertEquals(extractor.getErrorCode(), VideoInfo.NO_ERROR);
}
public void testGetErrorMessage() {
assertEquals(extractor.getErrorMessage(), "");
}
public void testGetTimeStamp() {
public void testGetInvalidTimeStamp() throws ParsingException {
assertTrue(Integer.toString(extractor.getTimeStamp()),
extractor.getTimeStamp() >= 0);
extractor.getTimeStamp() <= 0);
}
public void testGetTitle() {
public void testGetValidTimeStamp() throws CrawlingException, IOException {
YoutubeVideoExtractor extractor =
new YoutubeVideoExtractor("https://youtu.be/FmG385_uUys?t=174", new Downloader());
assertTrue(Integer.toString(extractor.getTimeStamp()),
extractor.getTimeStamp() == 174);
}
public void testGetTitle() throws ParsingException {
assertTrue(!extractor.getTitle().isEmpty());
}
public void testGetDescription() {
public void testGetDescription() throws ParsingException {
assertTrue(extractor.getDescription() != null);
}
public void testGetUploader() {
public void testGetUploader() throws ParsingException {
assertTrue(!extractor.getUploader().isEmpty());
}
public void testGetLength() {
public void testGetLength() throws ParsingException {
assertTrue(extractor.getLength() > 0);
}
public void testGetViews() {
public void testGetViews() throws ParsingException {
assertTrue(extractor.getLength() > 0);
}
public void testGetUploadDate() {
public void testGetUploadDate() throws ParsingException {
assertTrue(extractor.getUploadDate().length() > 0);
}
public void testGetThumbnailUrl() {
public void testGetThumbnailUrl() throws ParsingException {
assertTrue(extractor.getThumbnailUrl(),
extractor.getThumbnailUrl().contains("https://"));
}
public void testGetUploaderThumbnailUrl() {
public void testGetUploaderThumbnailUrl() throws ParsingException {
assertTrue(extractor.getUploaderThumbnailUrl(),
extractor.getUploaderThumbnailUrl().contains("https://"));
}
public void testGetAudioStreams() {
public void testGetAudioStreams() throws ParsingException {
for(VideoInfo.AudioStream s : extractor.getAudioStreams()) {
assertTrue(s.url,
s.url.contains("https://"));
@ -88,7 +93,7 @@ public class YoutubeVideoExtractorDefaultTest extends AndroidTestCase {
}
}
public void testGetVideoStreams() {
public void testGetVideoStreams() throws ParsingException {
for(VideoInfo.VideoStream s : extractor.getVideoStreams()) {
assertTrue(s.url,
s.url.contains("https://"));

40
app/src/androidTest/java/org/schabi/newpipe/services/youtube/YoutubeVideoExtractorGemaTest.java
View File

@ -2,7 +2,13 @@ package org.schabi.newpipe.services.youtube;
import android.test.AndroidTestCase;
import org.schabi.newpipe.services.VideoInfo;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.crawler.CrawlingException;
import org.schabi.newpipe.crawler.services.youtube.YoutubeVideoExtractor;
import org.schabi.newpipe.crawler.VideoInfo;
import org.schabi.newpipe.Downloader;
import java.io.IOException;
/**
* Created by the-scrabi on 30.12.15.
@ -29,31 +35,17 @@ import org.schabi.newpipe.services.VideoInfo;
public class YoutubeVideoExtractorGemaTest extends AndroidTestCase {
// Deaktivate this Test Case bevore uploading it githup, otherwise CI will fail.
private static final boolean testActive = false;
private static final boolean testActive = true;
private YoutubeVideoExtractor extractor;
public void setUp() {
public void testGemaError() throws IOException, CrawlingException {
if(testActive) {
extractor = new YoutubeVideoExtractor("https://www.youtube.com/watch?v=3O1_3zBUKM8");
}
}
public void testGetErrorCode() {
if(testActive) {
assertEquals(extractor.getErrorCode(), VideoInfo.ERROR_BLOCKED_BY_GEMA);
} else {
assertTrue(true);
}
}
public void testGetErrorMessage() {
if(testActive) {
assertTrue(extractor.getErrorMessage(),
extractor.getErrorMessage().contains("GEMA"));
} else {
assertTrue(true);
try {
new YoutubeVideoExtractor("https://www.youtube.com/watch?v=3O1_3zBUKM8",
new Downloader());
assertTrue("Gema exception not thrown", false);
} catch(YoutubeVideoExtractor.GemaException ge) {
assertTrue(true);
}
}
}
}

54
app/src/main/java/org/schabi/newpipe/Downloader.java
View File

@ -39,42 +39,39 @@ public class Downloader implements org.schabi.newpipe.crawler.Downloader {
* @param siteUrl the URL of the text file to return the contents of
* @param language the language (usually a 2-character code) to set as the preferred language
* @return the contents of the specified text file*/
public String download(String siteUrl, String language) {
String ret = "";
try {
URL url = new URL(siteUrl);
//HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
HttpsURLConnection con = NetCipher.getHttpsURLConnection(url);
con.setRequestProperty("Accept-Language", language);
ret = dl(con);
}
catch(Exception e) {
e.printStackTrace();
}
return ret;
public String download(String siteUrl, String language) throws IOException {
URL url = new URL(siteUrl);
//HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
HttpsURLConnection con = NetCipher.getHttpsURLConnection(url);
con.setRequestProperty("Accept-Language", language);
return dl(con);
}
/**Common functionality between download(String url) and download(String url, String language)*/
private static String dl(HttpsURLConnection con) throws IOException {
StringBuilder response = new StringBuilder();
BufferedReader in = null;
try {
con.setRequestMethod("GET");
con.setRequestProperty("User-Agent", USER_AGENT);
BufferedReader in = new BufferedReader(
in = new BufferedReader(
new InputStreamReader(con.getInputStream()));
String inputLine;
while((inputLine = in.readLine()) != null) {
response.append(inputLine);
}
in.close();
}
catch(UnknownHostException uhe) {//thrown when there's no internet connection
uhe.printStackTrace();
} catch(UnknownHostException uhe) {//thrown when there's no internet connection
throw new IOException("unknown host or no network", uhe);
//Toast.makeText(getActivity(), uhe.getMessage(), Toast.LENGTH_LONG).show();
} catch(Exception e) {
throw new IOException(e);
} finally {
if(in != null) {
in.close();
}
}
return response.toString();
@ -84,19 +81,10 @@ public class Downloader implements org.schabi.newpipe.crawler.Downloader {
* Primarily intended for downloading web pages.
* @param siteUrl the URL of the text file to download
* @return the contents of the specified text file*/
public String download(String siteUrl) {
String ret = "";
try {
URL url = new URL(siteUrl);
HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
//HttpsURLConnection con = NetCipher.getHttpsURLConnection(url);
ret = dl(con);
}
catch(Exception e) {
e.printStackTrace();
}
return ret;
public String download(String siteUrl) throws IOException {
URL url = new URL(siteUrl);
HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
//HttpsURLConnection con = NetCipher.getHttpsURLConnection(url);
return dl(con);
}
}

258
app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java
View File

@ -31,15 +31,20 @@ import android.widget.TextView;
import android.view.MenuItem;
import android.widget.Toast;
import java.io.IOException;
import java.net.URL;
import java.nio.charset.MalformedInputException;
import java.util.ArrayList;
import java.util.Vector;
import org.schabi.newpipe.crawler.CrawlingException;
import org.schabi.newpipe.crawler.ParsingException;
import org.schabi.newpipe.crawler.VideoPreviewInfo;
import org.schabi.newpipe.crawler.VideoExtractor;
import org.schabi.newpipe.crawler.ServiceList;
import org.schabi.newpipe.crawler.StreamingService;
import org.schabi.newpipe.crawler.VideoInfo;
import org.schabi.newpipe.crawler.services.youtube.YoutubeVideoExtractor;
/**
@ -68,7 +73,6 @@ public class VideoItemDetailFragment extends Fragment {
* The fragment argument representing the item ID that this fragment
* represents.
*/
//public static final String ARG_ITEM_ID = "item_id";
public static final String VIDEO_URL = "video_url";
public static final String STREAMING_SERVICE = "streaming_service";
public static final String AUTO_PLAY = "auto_play";
@ -87,7 +91,6 @@ public class VideoItemDetailFragment extends Fragment {
private FloatingActionButton playVideoButton;
private final Point initialThumbnailPos = new Point(0, 0);
public interface OnInvokeCreateOptionsMenuListener {
void createOptionsMenu();
}
@ -108,45 +111,64 @@ public class VideoItemDetailFragment extends Fragment {
@Override
public void run() {
try {
this.videoExtractor = service.getExtractorInstance(videoUrl, new Downloader());
videoExtractor = service.getExtractorInstance(videoUrl, new Downloader());
VideoInfo videoInfo = videoExtractor.getVideoInfo();
h.post(new VideoResultReturnedRunnable(videoInfo));
if (videoInfo.errorCode == VideoInfo.NO_ERROR) {
h.post(new SetThumbnailRunnable(
BitmapFactory.decodeStream(
new URL(videoInfo.thumbnail_url)
.openConnection()
.getInputStream()),
SetThumbnailRunnable.VIDEO_THUMBNAIL));
h.post(new SetThumbnailRunnable(
BitmapFactory.decodeStream(
new URL(videoInfo.uploader_thumbnail_url)
.openConnection()
.getInputStream()),
SetThumbnailRunnable.CHANNEL_THUMBNAIL));
if (showNextVideoItem) {
h.post(new SetThumbnailRunnable(
BitmapFactory.decodeStream(
new URL(videoInfo.thumbnail_url)
new URL(videoInfo.nextVideo.thumbnail_url)
.openConnection()
.getInputStream()),
SetThumbnailRunnable.VIDEO_THUMBNAIL));
h.post(new SetThumbnailRunnable(
BitmapFactory.decodeStream(
new URL(videoInfo.uploader_thumbnail_url)
.openConnection()
.getInputStream()),
SetThumbnailRunnable.CHANNEL_THUMBNAIL));
if(showNextVideoItem) {
h.post(new SetThumbnailRunnable(
BitmapFactory.decodeStream(
new URL(videoInfo.nextVideo.thumbnail_url)
.openConnection()
.getInputStream()),
SetThumbnailRunnable.NEXT_VIDEO_THUMBNAIL));
}
SetThumbnailRunnable.NEXT_VIDEO_THUMBNAIL));
}
} catch (Exception e) {
} catch (MalformedInputException e) {
postNewErrorToast(h, R.string.could_not_load_thumbnails);
e.printStackTrace();
} catch (IOException e) {
postNewErrorToast(h, R.string.network_error);
e.printStackTrace();
}
// custom service related exceptions
catch (YoutubeVideoExtractor.DecryptException de) {
postNewErrorToast(h, R.string.youtube_signature_decryption_error);
de.printStackTrace();
} catch (YoutubeVideoExtractor.GemaException ge) {
h.post(new Runnable() {
@Override
public void run() {
progressBar.setVisibility(View.GONE);
// This is poor style, but unless we have better error handling in the
// crawler, this may not be better.
Toast.makeText(VideoItemDetailFragment.this.getActivity(),
R.string.network_error, Toast.LENGTH_LONG).show();
onErrorBlockedByGema();
}
});
}
// ----------------------------------------
catch(VideoExtractor.ContentNotAvailableException e) {
h.post(new Runnable() {
@Override
public void run() {
onNotSpecifiedContentError();
}
});
e.printStackTrace();
} catch (ParsingException e) {
postNewErrorToast(h, e.getMessage());
e.printStackTrace();
} catch(Exception e) {
postNewErrorToast(h, R.string.general_error);
e.printStackTrace();
}
}
}
@ -213,7 +235,7 @@ public class VideoItemDetailFragment extends Fragment {
private void updateInfo(VideoInfo info) {
currentVideoInfo = info;
Resources res = activity.getResources();
try {
VideoInfoItemViewCreator videoItemViewCreator =
new VideoInfoItemViewCreator(LayoutInflater.from(getActivity()));
@ -226,107 +248,80 @@ public class VideoItemDetailFragment extends Fragment {
TextView thumbsDownView = (TextView) activity.findViewById(R.id.detailThumbsDownCountView);
TextView uploadDateView = (TextView) activity.findViewById(R.id.detailUploadDateView);
TextView descriptionView = (TextView) activity.findViewById(R.id.detailDescriptionView);
ImageView thumbnailView = (ImageView) activity.findViewById(R.id.detailThumbnailView);
FrameLayout nextVideoFrame = (FrameLayout) activity.findViewById(R.id.detailNextVideoFrame);
RelativeLayout nextVideoRootFrame =
(RelativeLayout) activity.findViewById(R.id.detailNextVideoRootLayout);
Button backgroundButton = (Button)
activity.findViewById(R.id.detailVideoThumbnailWindowBackgroundButton);
progressBar.setVisibility(View.GONE);
switch (info.errorCode) {
case VideoInfo.NO_ERROR: {
View nextVideoView = videoItemViewCreator
.getViewFromVideoInfoItem(null, nextVideoFrame, info.nextVideo, getContext());
nextVideoFrame.addView(nextVideoView);
View nextVideoView = videoItemViewCreator
.getViewFromVideoInfoItem(null, nextVideoFrame, info.nextVideo, getContext());
nextVideoFrame.addView(nextVideoView);
Button nextVideoButton = (Button) activity.findViewById(R.id.detailNextVideoButton);
Button similarVideosButton = (Button) activity.findViewById(R.id.detailShowSimilarButton);
Button nextVideoButton = (Button) activity.findViewById(R.id.detailNextVideoButton);
Button similarVideosButton = (Button) activity.findViewById(R.id.detailShowSimilarButton);
textContentLayout.setVisibility(View.VISIBLE);
playVideoButton.setVisibility(View.VISIBLE);
if (!showNextVideoItem) {
nextVideoRootFrame.setVisibility(View.GONE);
similarVideosButton.setVisibility(View.GONE);
}
textContentLayout.setVisibility(View.VISIBLE);
playVideoButton.setVisibility(View.VISIBLE);
if (!showNextVideoItem) {
nextVideoRootFrame.setVisibility(View.GONE);
similarVideosButton.setVisibility(View.GONE);
}
videoTitleView.setText(info.title);
uploaderView.setText(info.uploader);
actionBarHandler.setChannelName(info.uploader);
videoTitleView.setText(info.title);
uploaderView.setText(info.uploader);
actionBarHandler.setChannelName(info.uploader);
String localizedViewCount = Localization.localizeViewCount(info.view_count, getContext());
viewCountView.setText(localizedViewCount);
String localizedViewCount = Localization.localizeViewCount(info.view_count, getContext());
viewCountView.setText(localizedViewCount);
String localizedLikeCount = Localization.localizeNumber(info.like_count, getContext());
thumbsUpView.setText(localizedLikeCount);
String localizedLikeCount = Localization.localizeNumber(info.like_count, getContext());
thumbsUpView.setText(localizedLikeCount);
String localizedDislikeCount = Localization.localizeNumber(info.dislike_count, getContext());
thumbsDownView.setText(localizedDislikeCount);
String localizedDislikeCount = Localization.localizeNumber(info.dislike_count, getContext());
thumbsDownView.setText(localizedDislikeCount);
String localizedDate = Localization.localizeDate(info.upload_date, getContext());
uploadDateView.setText(localizedDate);
String localizedDate = Localization.localizeDate(info.upload_date, getContext());
uploadDateView.setText(localizedDate);
descriptionView.setText(Html.fromHtml(info.description));
descriptionView.setText(Html.fromHtml(info.description));
descriptionView.setMovementMethod(LinkMovementMethod.getInstance());
descriptionView.setMovementMethod(LinkMovementMethod.getInstance());
actionBarHandler.setServiceId(streamingServiceId);
actionBarHandler.setVideoInfo(info.webpage_url, info.title);
actionBarHandler.setStartPosition(info.startPosition);
actionBarHandler.setServiceId(streamingServiceId);
actionBarHandler.setVideoInfo(info.webpage_url, info.title);
actionBarHandler.setStartPosition(info.startPosition);
// parse streams
Vector<VideoInfo.VideoStream> streamsToUse = new Vector<>();
for (VideoInfo.VideoStream i : info.videoStreams) {
if (useStream(i, streamsToUse)) {
streamsToUse.add(i);
}
}
VideoInfo.VideoStream[] streamList = new VideoInfo.VideoStream[streamsToUse.size()];
for (int i = 0; i < streamList.length; i++) {
streamList[i] = streamsToUse.get(i);
}
actionBarHandler.setStreams(streamList, info.audioStreams);
// parse streams
Vector<VideoInfo.VideoStream> streamsToUse = new Vector<>();
for (VideoInfo.VideoStream i : info.videoStreams) {
if (useStream(i, streamsToUse)) {
streamsToUse.add(i);
}
}
VideoInfo.VideoStream[] streamList = new VideoInfo.VideoStream[streamsToUse.size()];
for (int i = 0; i < streamList.length; i++) {
streamList[i] = streamsToUse.get(i);
}
actionBarHandler.setStreams(streamList, info.audioStreams);
nextVideoButton.setOnClickListener(new View.OnClickListener() {
@Override
public void onClick(View v) {
Intent detailIntent =
new Intent(getActivity(), VideoItemDetailActivity.class);
nextVideoButton.setOnClickListener(new View.OnClickListener() {
@Override
public void onClick(View v) {
Intent detailIntent =
new Intent(getActivity(), VideoItemDetailActivity.class);
/*detailIntent.putExtra(
VideoItemDetailFragment.ARG_ITEM_ID, currentVideoInfo.nextVideo.id); */
detailIntent.putExtra(
VideoItemDetailFragment.VIDEO_URL, currentVideoInfo.nextVideo.webpage_url);
detailIntent.putExtra(
VideoItemDetailFragment.VIDEO_URL, currentVideoInfo.nextVideo.webpage_url);
detailIntent.putExtra(VideoItemDetailFragment.STREAMING_SERVICE, streamingServiceId);
startActivity(detailIntent);
}
});
detailIntent.putExtra(VideoItemDetailFragment.STREAMING_SERVICE, streamingServiceId);
startActivity(detailIntent);
}
break;
case VideoInfo.ERROR_BLOCKED_BY_GEMA:
thumbnailView.setImageBitmap(BitmapFactory.decodeResource(
getResources(), R.drawable.gruese_die_gema));
backgroundButton.setOnClickListener(new View.OnClickListener() {
@Override
public void onClick(View v) {
Intent intent = new Intent();
intent.setAction(Intent.ACTION_VIEW);
intent.setData(Uri.parse(activity.getString(R.string.c3s_url)));
activity.startActivity(intent);
}
});
break;
case VideoInfo.ERROR_NO_SPECIFIED_ERROR:
thumbnailView.setImageBitmap(BitmapFactory.decodeResource(
getResources(), R.drawable.not_available_monkey));
Toast.makeText(activity, info.errorMessage, Toast.LENGTH_LONG)
.show();
break;
default:
Log.e(TAG, "Video Available Status not known.");
}
});
if(autoPlayEnabled) {
actionBarHandler.playVideo();
@ -337,6 +332,37 @@ public class VideoItemDetailFragment extends Fragment {
}
}
private void onErrorBlockedByGema() {
Button backgroundButton = (Button)
activity.findViewById(R.id.detailVideoThumbnailWindowBackgroundButton);
ImageView thumbnailView = (ImageView) activity.findViewById(R.id.detailThumbnailView);
progressBar.setVisibility(View.GONE);
thumbnailView.setImageBitmap(BitmapFactory.decodeResource(
getResources(), R.drawable.gruese_die_gema));
backgroundButton.setOnClickListener(new View.OnClickListener() {
@Override
public void onClick(View v) {
Intent intent = new Intent();
intent.setAction(Intent.ACTION_VIEW);
intent.setData(Uri.parse(activity.getString(R.string.c3s_url)));
activity.startActivity(intent);
}
});
Toast.makeText(VideoItemDetailFragment.this.getActivity(),
R.string.blocked_by_gema, Toast.LENGTH_LONG).show();
}
private void onNotSpecifiedContentError() {
ImageView thumbnailView = (ImageView) activity.findViewById(R.id.detailThumbnailView);
progressBar.setVisibility(View.GONE);
thumbnailView.setImageBitmap(BitmapFactory.decodeResource(
getResources(), R.drawable.not_available_monkey));
Toast.makeText(activity, R.string.content_not_available, Toast.LENGTH_LONG)
.show();
}
private boolean useStream(VideoInfo.VideoStream stream, Vector<VideoInfo.VideoStream> streams) {
for(VideoInfo.VideoStream i : streams) {
if(i.resolution.equals(stream.resolution)) {
@ -465,4 +491,24 @@ public class VideoItemDetailFragment extends Fragment {
public void setOnInvokeCreateOptionsMenuListener(OnInvokeCreateOptionsMenuListener listener) {
this.onInvokeCreateOptionsMenuListener = listener;
}
private void postNewErrorToast(Handler h, final int stringResource) {
h.post(new Runnable() {
@Override
public void run() {
Toast.makeText(VideoItemDetailFragment.this.getActivity(),
stringResource, Toast.LENGTH_LONG).show();
}
});
}
private void postNewErrorToast(Handler h, final String message) {
h.post(new Runnable() {
@Override
public void run() {
Toast.makeText(VideoItemDetailFragment.this.getActivity(),
message, Toast.LENGTH_LONG).show();
}
});
}
}

30
app/src/main/java/org/schabi/newpipe/VideoItemListFragment.java
View File

@ -15,10 +15,12 @@ import android.widget.AbsListView;
import android.widget.ListView;
import android.widget.Toast;
import java.io.IOException;
import java.net.URL;
import java.util.List;
import java.util.Vector;
import org.schabi.newpipe.crawler.CrawlingException;
import org.schabi.newpipe.crawler.VideoPreviewInfo;
import org.schabi.newpipe.crawler.SearchEngine;
import org.schabi.newpipe.crawler.StreamingService;
@ -116,17 +118,15 @@ public class VideoItemListFragment extends ListFragment {
if(runs) {
h.post(new ResultRunnable(result, requestId));
}
} catch(Exception e) {
} catch(IOException e) {
postNewErrorToast(h, R.string.network_error);
e.printStackTrace();
} catch(CrawlingException ce) {
postNewErrorToast(h, R.string.parsing_error);
ce.printStackTrace();
} catch(Exception e) {
postNewErrorToast(h, R.string.general_error);
e.printStackTrace();
h.post(new Runnable() {
@Override
public void run() {
setListShown(true);
Toast.makeText(getActivity(), getString(R.string.network_error),
Toast.LENGTH_SHORT).show();
}
});
}
}
}
@ -386,4 +386,14 @@ public class VideoItemListFragment extends ListFragment {
mActivatedPosition = position;
}
private void postNewErrorToast(Handler h, final int stringResource) {
h.post(new Runnable() {
@Override
public void run() {
setListShown(true);
Toast.makeText(getActivity(), getString(R.string.network_error),
Toast.LENGTH_SHORT).show();
}
});
}
}

37
app/src/main/java/org/schabi/newpipe/crawler/CrawlingException.java Normal file
View File

@ -0,0 +1,37 @@
package org.schabi.newpipe.crawler;
/**
* Created by Christian Schabesberger on 30.01.16.
*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* CrawlingException.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class CrawlingException extends Exception {
public CrawlingException() {}
public CrawlingException(String message) {
super(message);
}
public CrawlingException(Throwable cause) {
super(cause);
}
public CrawlingException(String message, Throwable cause) {
super(message, cause);
}
}

12
app/src/main/java/org/schabi/newpipe/crawler/Downloader.java
View File

@ -1,5 +1,7 @@
package org.schabi.newpipe.crawler;
import java.io.IOException;
/**
* Created by Christian Schabesberger on 28.01.16.
*
@ -26,12 +28,14 @@ public interface Downloader {
* but set the HTTP header field "Accept-Language" to the supplied string.
* @param siteUrl the URL of the text file to return the contents of
* @param language the language (usually a 2-character code) to set as the preferred language
* @return the contents of the specified text file*/
String download(String siteUrl, String language);
* @return the contents of the specified text file
* @throws IOException*/
String download(String siteUrl, String language) throws IOException;
/**Download (via HTTP) the text file located at the supplied URL, and return its contents.
* Primarily intended for downloading web pages.
* @param siteUrl the URL of the text file to download
* @return the contents of the specified text file*/
String download(String siteUrl);
* @return the contents of the specified text file
* @throws IOException*/
String download(String siteUrl) throws IOException;
}

35
app/src/main/java/org/schabi/newpipe/crawler/ParsingException.java Normal file
View File

@ -0,0 +1,35 @@
package org.schabi.newpipe.crawler;
/**
* Created by Christian Schabesberger on 31.01.16.
*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* ParsingException.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class ParsingException extends CrawlingException {
public ParsingException() {}
public ParsingException(String message) {
super(message);
}
public ParsingException(Throwable cause) {
super(cause);
}
public ParsingException(String message, Throwable cause) {
super(message, cause);
}
}

7
app/src/main/java/org/schabi/newpipe/crawler/SearchEngine.java
View File

@ -1,5 +1,6 @@
package org.schabi.newpipe.crawler;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Vector;
@ -31,8 +32,10 @@ public interface SearchEngine {
public final Vector<VideoPreviewInfo> resultList = new Vector<>();
}
ArrayList<String> suggestionList(String query, Downloader dl);
ArrayList<String> suggestionList(String query, Downloader dl)
throws CrawlingException, IOException;
//Result search(String query, int page);
Result search(String query, int page, String contentCountry, Downloader dl);
Result search(String query, int page, String contentCountry, Downloader dl)
throws CrawlingException, IOException;
}

4
app/src/main/java/org/schabi/newpipe/crawler/StreamingService.java
View File

@ -1,5 +1,7 @@
package org.schabi.newpipe.crawler;
import java.io.IOException;
/**
* Created by Christian Schabesberger on 23.08.15.
*
@ -25,7 +27,7 @@ public interface StreamingService {
public String name = "";
}
ServiceInfo getServiceInfo();
VideoExtractor getExtractorInstance(String url, Downloader downloader);
VideoExtractor getExtractorInstance(String url, Downloader downloader) throws IOException, CrawlingException;
SearchEngine getSearchEngineInstance();
/**When a VIEW_ACTION is caught this function will test if the url delivered within the calling

178
app/src/main/java/org/schabi/newpipe/crawler/VideoExtractor.java
View File

@ -22,8 +22,43 @@ package org.schabi.newpipe.crawler;
/**Scrapes information from a video streaming service (eg, YouTube).*/
@SuppressWarnings("ALL")
public abstract class VideoExtractor {
public class ExctractorInitException extends CrawlingException {
public ExctractorInitException() {}
public ExctractorInitException(String message) {
super(message);
}
public ExctractorInitException(Throwable cause) {
super(cause);
}
public ExctractorInitException(String message, Throwable cause) {
super(message, cause);
}
}
public class RegexException extends ParsingException {
public RegexException() {}
public RegexException(String message) {
super(message);
}
}
public class ContentNotAvailableException extends ParsingException {
public ContentNotAvailableException() {}
public ContentNotAvailableException(String message) {
super(message);
}
public ContentNotAvailableException(Throwable cause) {
super(cause);
}
public ContentNotAvailableException(String message, Throwable cause) {
super(message, cause);
}
}
protected final String pageUrl;
protected VideoInfo videoInfo;
@ -34,7 +69,7 @@ public abstract class VideoExtractor {
/**Fills out the video info fields which are common to all services.
* Probably needs to be overridden by subclasses*/
public VideoInfo getVideoInfo()
public VideoInfo getVideoInfo() throws CrawlingException
{
if(videoInfo == null) {
videoInfo = new VideoInfo();
@ -44,90 +79,83 @@ public abstract class VideoExtractor {
videoInfo.webpage_url = pageUrl;
}
if(getErrorCode() == VideoInfo.NO_ERROR) {
if (videoInfo.title.isEmpty()) {
videoInfo.title = getTitle();
}
if (videoInfo.duration < 1) {
videoInfo.duration = getLength();
}
if (videoInfo.uploader.isEmpty()) {
videoInfo.uploader = getUploader();
}
if (videoInfo.description.isEmpty()) {
videoInfo.description = getDescription();
}
if (videoInfo.view_count == -1) {
videoInfo.view_count = getViews();
}
if (videoInfo.upload_date.isEmpty()) {
videoInfo.upload_date = getUploadDate();
}
if (videoInfo.thumbnail_url.isEmpty()) {
videoInfo.thumbnail_url = getThumbnailUrl();
}
if (videoInfo.id.isEmpty()) {
videoInfo.id = getVideoId(pageUrl);
}
/** Load and extract audio*/
if (videoInfo.audioStreams == null) {
videoInfo.audioStreams = getAudioStreams();
}
/** Extract video stream url*/
if (videoInfo.videoStreams == null) {
videoInfo.videoStreams = getVideoStreams();
}
if (videoInfo.uploader_thumbnail_url.isEmpty()) {
videoInfo.uploader_thumbnail_url = getUploaderThumbnailUrl();
}
if (videoInfo.startPosition < 0) {
videoInfo.startPosition = getTimeStamp();
}
if(videoInfo.dashMpdUrl.isEmpty()) {
videoInfo.dashMpdUrl = getDashMpdUrl();
}
} else {
videoInfo.errorCode = getErrorCode();
videoInfo.errorMessage = getErrorMessage();
if (videoInfo.title.isEmpty()) {
videoInfo.title = getTitle();
}
if (videoInfo.duration < 1) {
videoInfo.duration = getLength();
}
if (videoInfo.uploader.isEmpty()) {
videoInfo.uploader = getUploader();
}
if (videoInfo.description.isEmpty()) {
videoInfo.description = getDescription();
}
if (videoInfo.view_count == -1) {
videoInfo.view_count = getViews();
}
if (videoInfo.upload_date.isEmpty()) {
videoInfo.upload_date = getUploadDate();
}
if (videoInfo.thumbnail_url.isEmpty()) {
videoInfo.thumbnail_url = getThumbnailUrl();
}
if (videoInfo.id.isEmpty()) {
videoInfo.id = getVideoId(pageUrl);
}
/** Load and extract audio*/
if (videoInfo.audioStreams == null) {
videoInfo.audioStreams = getAudioStreams();
}
/** Extract video stream url*/
if (videoInfo.videoStreams == null) {
videoInfo.videoStreams = getVideoStreams();
}
if (videoInfo.uploader_thumbnail_url.isEmpty()) {
videoInfo.uploader_thumbnail_url = getUploaderThumbnailUrl();
}
if (videoInfo.startPosition < 0) {
videoInfo.startPosition = getTimeStamp();
}
if(videoInfo.dashMpdUrl.isEmpty()) {
videoInfo.dashMpdUrl = getDashMpdUrl();
}
//Bitmap thumbnail = null;
//Bitmap uploader_thumbnail = null;
//int videoAvailableStatus = VIDEO_AVAILABLE;
return videoInfo;
}
//todo: add licence field
public abstract int getErrorCode();
public abstract String getErrorMessage();
//todo: remove these functions, or make them static, otherwise its useles, to have them here
public abstract String getVideoUrl(String videoId);
public abstract String getVideoId(String siteUrl);
public abstract String getVideoId(String siteUrl) throws ParsingException;
///////////////////////////////////////////////////////////////////////////////////////////
public abstract int getTimeStamp();
public abstract String getTitle();
public abstract String getDescription();
public abstract String getUploader();
public abstract int getLength();
public abstract long getViews();
public abstract String getUploadDate();
public abstract String getThumbnailUrl();
public abstract String getUploaderThumbnailUrl();
public abstract VideoInfo.AudioStream[] getAudioStreams();
public abstract VideoInfo.VideoStream[] getVideoStreams();
public abstract String getDashMpdUrl();
public abstract int getTimeStamp() throws ParsingException;
public abstract String getTitle() throws ParsingException;
public abstract String getDescription() throws ParsingException;
public abstract String getUploader() throws ParsingException;
public abstract int getLength() throws ParsingException;
public abstract long getViews() throws ParsingException;
public abstract String getUploadDate() throws ParsingException;
public abstract String getThumbnailUrl() throws ParsingException;
public abstract String getUploaderThumbnailUrl() throws ParsingException;
public abstract VideoInfo.AudioStream[] getAudioStreams() throws ParsingException;
public abstract VideoInfo.VideoStream[] getVideoStreams() throws ParsingException;
public abstract String getDashMpdUrl() throws ParsingException;
public abstract int getAgeLimit() throws ParsingException;
}

11
app/src/main/java/org/schabi/newpipe/crawler/VideoInfo.java
View File

@ -26,14 +26,6 @@ import java.util.List;
@SuppressWarnings("ALL")
public class VideoInfo extends AbstractVideoInfo {
// If a video could not be parsed, this predefined error codes
// will be returned AND can be parsed by the frontend of the app.
// Error codes:
public final static int NO_ERROR = 0x0;
public final static int ERROR_NO_SPECIFIED_ERROR = 0x1;
// GEMA a german music colecting society.
public final static int ERROR_BLOCKED_BY_GEMA = 0x2;
public String uploader_thumbnail_url = "";
public String description = "";
public VideoStream[] videoStreams = null;
@ -43,8 +35,6 @@ public class VideoInfo extends AbstractVideoInfo {
// crawling such a file is not service dependent. Therefore getting audio only streams by yust
// providing the dash mpd fille will be possible in the future.
public String dashMpdUrl = "";
public int errorCode = NO_ERROR;
public String errorMessage = "";
public int duration = -1;
/*YouTube-specific fields
@ -60,7 +50,6 @@ public class VideoInfo extends AbstractVideoInfo {
public VideoInfo() {}
/**Creates a new VideoInfo object from an existing AbstractVideoInfo.
* All the shared properties are copied to the new VideoInfo.*/
@SuppressWarnings("WeakerAccess")

162
app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeSearchEngine.java
View File

@ -6,8 +6,11 @@ import android.util.Log;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.schabi.newpipe.crawler.CrawlingException;
import org.schabi.newpipe.crawler.Downloader;
import org.schabi.newpipe.crawler.ParsingException;
import org.schabi.newpipe.crawler.SearchEngine;
import org.schabi.newpipe.crawler.VideoExtractor;
import org.schabi.newpipe.crawler.VideoPreviewInfo;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
@ -49,8 +52,8 @@ public class YoutubeSearchEngine implements SearchEngine {
private static final String TAG = YoutubeSearchEngine.class.toString();
@Override
public Result search(String query, int page, String languageCode, Downloader downloader) {
//String contentCountry = PreferenceManager.getDefaultSharedPreferences(this).getString(getString(R.string., "");
public Result search(String query, int page, String languageCode, Downloader downloader) throws IOException, ParsingException {
Result result = new Result();
Uri.Builder builder = new Uri.Builder();
builder.scheme("https")
.authority("www.youtube.com")
@ -63,22 +66,19 @@ public class YoutubeSearchEngine implements SearchEngine {
String url = builder.build().toString();
//if we've been passed a valid language code, append it to the URL
if(!languageCode.isEmpty()) {
//assert Pattern.matches("[a-z]{2}(-([A-Z]{2}|[0-9]{1,3}))?", languageCode);
site = downloader.download(url, languageCode);
//assert Pattern.matches("[a-z]{2}(-([A-Z]{2}|[0-9]{1,3}))?", languageCode);
site = downloader.download(url, languageCode);
}
else {
site = downloader.download(url);
}
try {
Document doc = Jsoup.parse(site, url);
Result result = new Result();
Element list = doc.select("ol[class=\"item-section\"]").first();
Document doc = Jsoup.parse(site, url);
Element list = doc.select("ol[class=\"item-section\"]").first();
int i = 0;
for(Element item : list.children()) {
i++;
for (Element item : list.children()) {
/* First we need to determine which kind of item we are working with.
Youtube depicts five different kinds of items on its search result page. These are
regular videos, playlists, channels, two types of video suggestions, and a "no video
@ -90,57 +90,61 @@ public class YoutubeSearchEngine implements SearchEngine {
playlists now.
*/
Element el;
Element el;
// both types of spell correction item
if(!((el = item.select("div[class*=\"spell-correction\"]").first()) == null)) {
result.suggestion = el.select("a").first().text();
// search message item
} else if(!((el = item.select("div[class*=\"search-message\"]").first()) == null)) {
result.errorMessage = el.text();
// both types of spell correction item
if (!((el = item.select("div[class*=\"spell-correction\"]").first()) == null)) {
result.suggestion = el.select("a").first().text();
// search message item
} else if (!((el = item.select("div[class*=\"search-message\"]").first()) == null)) {
result.errorMessage = el.text();
// video item type
} else if(!((el = item.select("div[class*=\"yt-lockup-video\"").first()) == null)) {
VideoPreviewInfo resultItem = new VideoPreviewInfo();
Element dl = el.select("h3").first().select("a").first();
resultItem.webpage_url = dl.attr("abs:href");
try {
Pattern p = Pattern.compile("v=([0-9a-zA-Z-]*)");
Matcher m = p.matcher(resultItem.webpage_url);
resultItem.id=m.group(1);
} catch (Exception e) {
//e.printStackTrace();
// video item type
} else if (!((el = item.select("div[class*=\"yt-lockup-video\"").first()) == null)) {
VideoPreviewInfo resultItem = new VideoPreviewInfo();
Element dl = el.select("h3").first().select("a").first();
resultItem.webpage_url = dl.attr("abs:href");
try {
Pattern p = Pattern.compile("v=([0-9a-zA-Z-]*)");
Matcher m = p.matcher(resultItem.webpage_url);
resultItem.id = m.group(1);
} catch (Exception e) {
//e.printStackTrace();
}
resultItem.title = dl.text();
resultItem.duration = item.select("span[class=\"video-time\"]").first().text();
resultItem.uploader = item.select("div[class=\"yt-lockup-byline\"]").first()
.select("a").first()
.text();
resultItem.upload_date = item.select("div[class=\"yt-lockup-meta\"]").first()
.select("li").first()
.text();
Element te = item.select("div[class=\"yt-thumb video-thumb\"]").first()
.select("img").first();
resultItem.thumbnail_url = te.attr("abs:src");
// Sometimes youtube sends links to gif files which somehow seem to not exist
// anymore. Items with such gif also offer a secondary image source. So we are going
// to use that if we've caught such an item.
if (resultItem.thumbnail_url.contains(".gif")) {
resultItem.thumbnail_url = te.attr("abs:data-thumb");
}
result.resultList.add(resultItem);
} else {
//noinspection ConstantConditions
Log.e(TAG, "unexpected element found:\"" + el + "\"");
}
resultItem.title = dl.text();
resultItem.duration = item.select("span[class=\"video-time\"]").first().text();
resultItem.uploader = item.select("div[class=\"yt-lockup-byline\"]").first()
.select("a").first()
.text();
resultItem.upload_date = item.select("div[class=\"yt-lockup-meta\"]").first()
.select("li").first()
.text();
Element te = item.select("div[class=\"yt-thumb video-thumb\"]").first()
.select("img").first();
resultItem.thumbnail_url = te.attr("abs:src");
// Sometimes youtube sends links to gif files which somehow seem to not exist
// anymore. Items with such gif also offer a secondary image source. So we are going
// to use that if we've caught such an item.
if(resultItem.thumbnail_url.contains(".gif")) {
resultItem.thumbnail_url = te.attr("abs:data-thumb");
}
result.resultList.add(resultItem);
} else {
//noinspection ConstantConditions
Log.e(TAG, "unexpected element found:\""+el+"\"");
}
} catch(Exception e) {
throw new ParsingException(e);
}
return result;
}
@Override
public ArrayList<String> suggestionList(String query, Downloader dl) {
public ArrayList<String> suggestionList(String query, Downloader dl)
throws IOException, ParsingException {
ArrayList<String> suggestions = new ArrayList<>();
@ -155,36 +159,42 @@ public class YoutubeSearchEngine implements SearchEngine {
.appendQueryParameter("q", query);
String url = builder.build().toString();
String response = dl.download(url);
//TODO: Parse xml data using Jsoup not done
DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
DocumentBuilder dBuilder;
org.w3c.dom.Document doc = null;
try {
dBuilder = dbFactory.newDocumentBuilder();
doc = dBuilder.parse(new InputSource(new ByteArrayInputStream(response.getBytes("utf-8"))));
doc.getDocumentElement().normalize();
}catch (ParserConfigurationException | SAXException | IOException e) {
e.printStackTrace();
}
if(doc!=null){
NodeList nList = doc.getElementsByTagName("CompleteSuggestion");
for (int temp = 0; temp < nList.getLength(); temp++) {
//TODO: Parse xml data using Jsoup not done
DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
DocumentBuilder dBuilder;
org.w3c.dom.Document doc = null;
NodeList nList1 = doc.getElementsByTagName("suggestion");
Node nNode1 = nList1.item(temp);
if (nNode1.getNodeType() == Node.ELEMENT_NODE) {
org.w3c.dom.Element eElement = (org.w3c.dom.Element) nNode1;
suggestions.add(eElement.getAttribute("data"));
}
try {
dBuilder = dbFactory.newDocumentBuilder();
doc = dBuilder.parse(new InputSource(new ByteArrayInputStream(response.getBytes("utf-8"))));
doc.getDocumentElement().normalize();
} catch (ParserConfigurationException | SAXException | IOException e) {
e.printStackTrace();
}
}else {
Log.e(TAG, "GREAT FUCKING ERROR");
if (doc != null) {
NodeList nList = doc.getElementsByTagName("CompleteSuggestion");
for (int temp = 0; temp < nList.getLength(); temp++) {
NodeList nList1 = doc.getElementsByTagName("suggestion");
Node nNode1 = nList1.item(temp);
if (nNode1.getNodeType() == Node.ELEMENT_NODE) {
org.w3c.dom.Element eElement = (org.w3c.dom.Element) nNode1;
suggestions.add(eElement.getAttribute("data"));
}
}
} else {
Log.e(TAG, "GREAT FUCKING ERROR");
}
return suggestions;
} catch(Exception e) {
throw new ParsingException(e);
}
return suggestions;
}
}

7
app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeService.java
View File

@ -1,10 +1,13 @@
package org.schabi.newpipe.crawler.services.youtube;
import org.schabi.newpipe.crawler.CrawlingException;
import org.schabi.newpipe.crawler.Downloader;
import org.schabi.newpipe.crawler.StreamingService;
import org.schabi.newpipe.crawler.VideoExtractor;
import org.schabi.newpipe.crawler.SearchEngine;
import java.io.IOException;
/**
* Created by Christian Schabesberger on 23.08.15.
@ -34,9 +37,9 @@ public class YoutubeService implements StreamingService {
return serviceInfo;
}
@Override
public VideoExtractor getExtractorInstance(String url, Downloader downloader) {
public VideoExtractor getExtractorInstance(String url, Downloader downloader) throws CrawlingException, IOException {
if(acceptUrl(url)) {
return new YoutubeVideoExtractor(url, downloader);
return new YoutubeVideoExtractor(url, downloader) ;
}
else {
throw new IllegalArgumentException("supplied String is not a valid Youtube URL");

647
app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeVideoExtractor.java
View File

File diff suppressed because it is too large Load Diff

6
app/src/main/res/values/strings.xml
View File

@ -65,7 +65,13 @@
<string name="background_player_playing_toast">Playing in background</string>
<string name="c3s_url" translatable="false">https://www.c3s.cc/</string>
<string name="play_btn_text">Play</string>
<string name="general_error">Error</string>
<string name="network_error">Network error</string>
<string name="could_not_load_thumbnails">Could not load Thumbnails</string>
<string name="youtube_signature_decryption_error">Could not decrypt video url signature.</string>
<string name="parsing_error">Could not parse website.</string>
<string name="content_not_available">Content not available.</string>
<string name="blocked_by_gema">Blocked by GEMA.</string>
<!-- Content descriptions (for better accessibility) -->
<string name="list_thumbnail_view_description">Video preview thumbnail</string>