diff --git a/app/build.gradle b/app/build.gradle index b385015f7..78ac4f3e3 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -54,7 +54,7 @@ dependencies { exclude module: 'support-annotations' }) - implementation 'com.github.TeamNewPipe:NewPipeExtractor:32d316330c26' + implementation 'com.github.TeamNewPipe:NewPipeExtractor:91b1efc97e' testImplementation 'junit:junit:4.12' testImplementation 'org.mockito:mockito-core:2.23.0' diff --git a/app/src/main/java/org/schabi/newpipe/Downloader.java b/app/src/main/java/org/schabi/newpipe/Downloader.java index 62c7d1671..32e8bd414 100644 --- a/app/src/main/java/org/schabi/newpipe/Downloader.java +++ b/app/src/main/java/org/schabi/newpipe/Downloader.java @@ -89,7 +89,8 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader { .build(); response = client.newCall(request).execute(); - return Long.parseLong(response.header("Content-Length")); + String contentLength = response.header("Content-Length"); + return contentLength == null ? -1 : Long.parseLong(contentLength); } catch (NumberFormatException e) { throw new IOException("Invalid content length", e); } finally { diff --git a/app/src/main/java/org/schabi/newpipe/download/DeleteDownloadManager.java b/app/src/main/java/org/schabi/newpipe/download/DeleteDownloadManager.java deleted file mode 100644 index 5a2d4a486..000000000 --- a/app/src/main/java/org/schabi/newpipe/download/DeleteDownloadManager.java +++ /dev/null @@ -1,158 +0,0 @@ -package org.schabi.newpipe.download; - -import android.app.Activity; -import android.os.Bundle; -import android.support.annotation.NonNull; -import android.support.annotation.Nullable; -import android.support.design.widget.BaseTransientBottomBar; -import android.support.design.widget.Snackbar; -import android.view.View; - -import org.schabi.newpipe.R; - -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.concurrent.TimeUnit; - -import io.reactivex.Completable; -import io.reactivex.Observable; -import io.reactivex.android.schedulers.AndroidSchedulers; -import io.reactivex.disposables.Disposable; -import io.reactivex.schedulers.Schedulers; -import io.reactivex.subjects.PublishSubject; -import us.shandian.giga.get.DownloadManager; -import us.shandian.giga.get.DownloadMission; - -public class DeleteDownloadManager { - - private static final String KEY_STATE = "delete_manager_state"; - - private final View mView; - private final HashSet mPendingMap; - private final List mDisposableList; - private DownloadManager mDownloadManager; - private final PublishSubject publishSubject = PublishSubject.create(); - - DeleteDownloadManager(Activity activity) { - mPendingMap = new HashSet<>(); - mDisposableList = new ArrayList<>(); - mView = activity.findViewById(android.R.id.content); - } - - public Observable getUndoObservable() { - return publishSubject; - } - - public boolean contains(@NonNull DownloadMission mission) { - return mPendingMap.contains(mission.url); - } - - public void add(@NonNull DownloadMission mission) { - mPendingMap.add(mission.url); - - if (mPendingMap.size() == 1) { - showUndoDeleteSnackbar(mission); - } - } - - public void setDownloadManager(@NonNull DownloadManager downloadManager) { - mDownloadManager = downloadManager; - - if (mPendingMap.size() < 1) return; - - showUndoDeleteSnackbar(); - } - - public void restoreState(@Nullable Bundle savedInstanceState) { - if (savedInstanceState == null) return; - - List list = savedInstanceState.getStringArrayList(KEY_STATE); - if (list != null) { - mPendingMap.addAll(list); - } - } - - public void saveState(@Nullable Bundle outState) { - if (outState == null) return; - - for (Disposable disposable : mDisposableList) { - disposable.dispose(); - } - - outState.putStringArrayList(KEY_STATE, new ArrayList<>(mPendingMap)); - } - - private void showUndoDeleteSnackbar() { - if (mPendingMap.size() < 1) return; - - String url = mPendingMap.iterator().next(); - - for (int i = 0; i < mDownloadManager.getCount(); i++) { - DownloadMission mission = mDownloadManager.getMission(i); - if (url.equals(mission.url)) { - showUndoDeleteSnackbar(mission); - break; - } - } - } - - private void showUndoDeleteSnackbar(@NonNull DownloadMission mission) { - final Snackbar snackbar = Snackbar.make(mView, mission.name, Snackbar.LENGTH_INDEFINITE); - final Disposable disposable = Observable.timer(3, TimeUnit.SECONDS) - .subscribeOn(AndroidSchedulers.mainThread()) - .subscribe(l -> snackbar.dismiss()); - - mDisposableList.add(disposable); - - snackbar.setAction(R.string.undo, v -> { - mPendingMap.remove(mission.url); - publishSubject.onNext(mission); - disposable.dispose(); - snackbar.dismiss(); - }); - - snackbar.addCallback(new BaseTransientBottomBar.BaseCallback() { - @Override - public void onDismissed(Snackbar transientBottomBar, int event) { - if (!disposable.isDisposed()) { - Completable.fromAction(() -> deletePending(mission)) - .subscribeOn(Schedulers.io()) - .subscribe(); - } - mPendingMap.remove(mission.url); - snackbar.removeCallback(this); - mDisposableList.remove(disposable); - showUndoDeleteSnackbar(); - } - }); - - snackbar.show(); - } - - public void deletePending() { - if (mPendingMap.size() < 1) return; - - HashSet idSet = new HashSet<>(); - for (int i = 0; i < mDownloadManager.getCount(); i++) { - if (contains(mDownloadManager.getMission(i))) { - idSet.add(i); - } - } - - for (Integer id : idSet) { - mDownloadManager.deleteMission(id); - } - - mPendingMap.clear(); - } - - private void deletePending(@NonNull DownloadMission mission) { - for (int i = 0; i < mDownloadManager.getCount(); i++) { - if (mission.url.equals(mDownloadManager.getMission(i).url)) { - mDownloadManager.deleteMission(i); - break; - } - } - } -} diff --git a/app/src/main/java/org/schabi/newpipe/download/DownloadActivity.java b/app/src/main/java/org/schabi/newpipe/download/DownloadActivity.java index 4a2c85149..251e4c730 100644 --- a/app/src/main/java/org/schabi/newpipe/download/DownloadActivity.java +++ b/app/src/main/java/org/schabi/newpipe/download/DownloadActivity.java @@ -15,16 +15,12 @@ import org.schabi.newpipe.R; import org.schabi.newpipe.settings.SettingsActivity; import org.schabi.newpipe.util.ThemeHelper; -import io.reactivex.Completable; -import io.reactivex.schedulers.Schedulers; import us.shandian.giga.service.DownloadManagerService; -import us.shandian.giga.ui.fragment.AllMissionsFragment; import us.shandian.giga.ui.fragment.MissionsFragment; public class DownloadActivity extends AppCompatActivity { private static final String MISSIONS_FRAGMENT_TAG = "fragment_tag"; - private DeleteDownloadManager mDeleteDownloadManager; @Override protected void onCreate(Bundle savedInstanceState) { @@ -47,32 +43,17 @@ public class DownloadActivity extends AppCompatActivity { actionBar.setDisplayShowTitleEnabled(true); } - mDeleteDownloadManager = new DeleteDownloadManager(this); - mDeleteDownloadManager.restoreState(savedInstanceState); - - MissionsFragment fragment = (MissionsFragment) getFragmentManager().findFragmentByTag(MISSIONS_FRAGMENT_TAG); - if (fragment != null) { - fragment.setDeleteManager(mDeleteDownloadManager); - } else { - getWindow().getDecorView().getViewTreeObserver().addOnGlobalLayoutListener(new ViewTreeObserver.OnGlobalLayoutListener() { - @Override - public void onGlobalLayout() { - updateFragments(); - getWindow().getDecorView().getViewTreeObserver().removeGlobalOnLayoutListener(this); - } - }); - } - } - - @Override - protected void onSaveInstanceState(Bundle outState) { - mDeleteDownloadManager.saveState(outState); - super.onSaveInstanceState(outState); + getWindow().getDecorView().getViewTreeObserver().addOnGlobalLayoutListener(new ViewTreeObserver.OnGlobalLayoutListener() { + @Override + public void onGlobalLayout() { + updateFragments(); + getWindow().getDecorView().getViewTreeObserver().removeGlobalOnLayoutListener(this); + } + }); } private void updateFragments() { - MissionsFragment fragment = new AllMissionsFragment(); - fragment.setDeleteManager(mDeleteDownloadManager); + MissionsFragment fragment = new MissionsFragment(); getFragmentManager().beginTransaction() .replace(R.id.frame, fragment, MISSIONS_FRAGMENT_TAG) @@ -99,7 +80,6 @@ public class DownloadActivity extends AppCompatActivity { case R.id.action_settings: { Intent intent = new Intent(this, SettingsActivity.class); startActivity(intent); - deletePending(); return true; } default: @@ -108,14 +88,7 @@ public class DownloadActivity extends AppCompatActivity { } @Override - public void onBackPressed() { - super.onBackPressed(); - deletePending(); - } - - private void deletePending() { - Completable.fromAction(mDeleteDownloadManager::deletePending) - .subscribeOn(Schedulers.io()) - .subscribe(); + public void onRestoreInstanceState(Bundle inState){ + super.onRestoreInstanceState(inState); } } diff --git a/app/src/main/java/org/schabi/newpipe/download/DownloadDialog.java b/app/src/main/java/org/schabi/newpipe/download/DownloadDialog.java index 9bbda6032..4f98f7f28 100644 --- a/app/src/main/java/org/schabi/newpipe/download/DownloadDialog.java +++ b/app/src/main/java/org/schabi/newpipe/download/DownloadDialog.java @@ -1,13 +1,17 @@ package org.schabi.newpipe.download; import android.content.Context; +import android.content.SharedPreferences; import android.os.Bundle; +import android.preference.PreferenceManager; import android.support.annotation.IdRes; import android.support.annotation.NonNull; import android.support.annotation.Nullable; import android.support.v4.app.DialogFragment; +import android.support.v7.app.AlertDialog; import android.support.v7.widget.Toolbar; import android.util.Log; +import android.util.SparseArray; import android.view.LayoutInflater; import android.view.View; import android.view.ViewGroup; @@ -22,38 +26,55 @@ import android.widget.Toast; import org.schabi.newpipe.MainActivity; import org.schabi.newpipe.R; +import org.schabi.newpipe.extractor.MediaFormat; +import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.stream.AudioStream; import org.schabi.newpipe.extractor.stream.Stream; import org.schabi.newpipe.extractor.stream.StreamInfo; +import org.schabi.newpipe.extractor.stream.SubtitlesStream; import org.schabi.newpipe.extractor.stream.VideoStream; +import org.schabi.newpipe.extractor.utils.Localization; import org.schabi.newpipe.settings.NewPipeSettings; import org.schabi.newpipe.util.FilenameUtils; import org.schabi.newpipe.util.ListHelper; import org.schabi.newpipe.util.PermissionHelper; +import org.schabi.newpipe.util.SecondaryStreamHelper; import org.schabi.newpipe.util.StreamItemAdapter; import org.schabi.newpipe.util.StreamItemAdapter.StreamSizeWrapper; import org.schabi.newpipe.util.ThemeHelper; import java.util.ArrayList; import java.util.List; +import java.util.Locale; import icepick.Icepick; import icepick.State; import io.reactivex.disposables.CompositeDisposable; +import us.shandian.giga.postprocessing.Postprocessing; import us.shandian.giga.service.DownloadManagerService; public class DownloadDialog extends DialogFragment implements RadioGroup.OnCheckedChangeListener, AdapterView.OnItemSelectedListener { private static final String TAG = "DialogFragment"; private static final boolean DEBUG = MainActivity.DEBUG; - @State protected StreamInfo currentInfo; - @State protected StreamSizeWrapper wrappedAudioStreams = StreamSizeWrapper.empty(); - @State protected StreamSizeWrapper wrappedVideoStreams = StreamSizeWrapper.empty(); - @State protected int selectedVideoIndex = 0; - @State protected int selectedAudioIndex = 0; + @State + protected StreamInfo currentInfo; + @State + protected StreamSizeWrapper wrappedAudioStreams = StreamSizeWrapper.empty(); + @State + protected StreamSizeWrapper wrappedVideoStreams = StreamSizeWrapper.empty(); + @State + protected StreamSizeWrapper wrappedSubtitleStreams = StreamSizeWrapper.empty(); + @State + protected int selectedVideoIndex = 0; + @State + protected int selectedAudioIndex = 0; + @State + protected int selectedSubtitleIndex = 0; - private StreamItemAdapter audioStreamsAdapter; - private StreamItemAdapter videoStreamsAdapter; + private StreamItemAdapter audioStreamsAdapter; + private StreamItemAdapter videoStreamsAdapter; + private StreamItemAdapter subtitleStreamsAdapter; private final CompositeDisposable disposables = new CompositeDisposable(); @@ -63,6 +84,8 @@ public class DownloadDialog extends DialogFragment implements RadioGroup.OnCheck private TextView threadsCountTextView; private SeekBar threadsSeekBar; + private SharedPreferences prefs; + public static DownloadDialog newInstance(StreamInfo info) { DownloadDialog dialog = new DownloadDialog(); dialog.setInfo(info); @@ -78,6 +101,8 @@ public class DownloadDialog extends DialogFragment implements RadioGroup.OnCheck instance.setVideoStreams(streamsList); instance.setSelectedVideoStream(selectedStreamIndex); instance.setAudioStreams(info.getAudioStreams()); + instance.setSubtitleStreams(info.getSubtitles()); + return instance; } @@ -86,7 +111,7 @@ public class DownloadDialog extends DialogFragment implements RadioGroup.OnCheck } public void setAudioStreams(List audioStreams) { - setAudioStreams(new StreamSizeWrapper<>(audioStreams)); + setAudioStreams(new StreamSizeWrapper<>(audioStreams, getContext())); } public void setAudioStreams(StreamSizeWrapper wrappedAudioStreams) { @@ -94,13 +119,21 @@ public class DownloadDialog extends DialogFragment implements RadioGroup.OnCheck } public void setVideoStreams(List videoStreams) { - setVideoStreams(new StreamSizeWrapper<>(videoStreams)); + setVideoStreams(new StreamSizeWrapper<>(videoStreams, getContext())); } public void setVideoStreams(StreamSizeWrapper wrappedVideoStreams) { this.wrappedVideoStreams = wrappedVideoStreams; } + public void setSubtitleStreams(List subtitleStreams) { + setSubtitleStreams(new StreamSizeWrapper<>(subtitleStreams, getContext())); + } + + public void setSubtitleStreams(StreamSizeWrapper wrappedSubtitleStreams) { + this.wrappedSubtitleStreams = wrappedSubtitleStreams; + } + public void setSelectedVideoStream(int selectedVideoIndex) { this.selectedVideoIndex = selectedVideoIndex; } @@ -109,6 +142,10 @@ public class DownloadDialog extends DialogFragment implements RadioGroup.OnCheck this.selectedAudioIndex = selectedAudioIndex; } + public void setSelectedSubtitleStream(int selectedSubtitleIndex) { + this.selectedSubtitleIndex = selectedSubtitleIndex; + } + /*////////////////////////////////////////////////////////////////////////// // LifeCycle //////////////////////////////////////////////////////////////////////////*/ @@ -116,7 +153,8 @@ public class DownloadDialog extends DialogFragment implements RadioGroup.OnCheck @Override public void onCreate(@Nullable Bundle savedInstanceState) { super.onCreate(savedInstanceState); - if (DEBUG) Log.d(TAG, "onCreate() called with: savedInstanceState = [" + savedInstanceState + "]"); + if (DEBUG) + Log.d(TAG, "onCreate() called with: savedInstanceState = [" + savedInstanceState + "]"); if (!PermissionHelper.checkStoragePermissions(getActivity(), PermissionHelper.DOWNLOAD_DIALOG_REQUEST_CODE)) { getDialog().dismiss(); return; @@ -125,13 +163,29 @@ public class DownloadDialog extends DialogFragment implements RadioGroup.OnCheck setStyle(STYLE_NO_TITLE, ThemeHelper.getDialogTheme(getContext())); Icepick.restoreInstanceState(this, savedInstanceState); - this.videoStreamsAdapter = new StreamItemAdapter<>(getContext(), wrappedVideoStreams, true); + SparseArray> secondaryStreams = new SparseArray<>(4); + List videoStreams = wrappedVideoStreams.getStreamsList(); + + for (int i = 0; i < videoStreams.size(); i++) { + if (!videoStreams.get(i).isVideoOnly()) continue; + AudioStream audioStream = SecondaryStreamHelper.getAudioStreamFor(wrappedAudioStreams.getStreamsList(), videoStreams.get(i)); + + if (audioStream != null) { + secondaryStreams.append(i, new SecondaryStreamHelper<>(wrappedAudioStreams, audioStream)); + } else if (DEBUG) { + Log.w(TAG, "No audio stream candidates for video format " + videoStreams.get(i).getFormat().name()); + } + } + + this.videoStreamsAdapter = new StreamItemAdapter<>(getContext(), wrappedVideoStreams, secondaryStreams); this.audioStreamsAdapter = new StreamItemAdapter<>(getContext(), wrappedAudioStreams); + this.subtitleStreamsAdapter = new StreamItemAdapter<>(getContext(), wrappedSubtitleStreams); } @Override public View onCreateView(@NonNull LayoutInflater inflater, ViewGroup container, Bundle savedInstanceState) { - if (DEBUG) Log.d(TAG, "onCreateView() called with: inflater = [" + inflater + "], container = [" + container + "], savedInstanceState = [" + savedInstanceState + "]"); + if (DEBUG) + Log.d(TAG, "onCreateView() called with: inflater = [" + inflater + "], container = [" + container + "], savedInstanceState = [" + savedInstanceState + "]"); return inflater.inflate(R.layout.download_dialog, container); } @@ -142,6 +196,8 @@ public class DownloadDialog extends DialogFragment implements RadioGroup.OnCheck nameEditText.setText(FilenameUtils.createFilename(getContext(), currentInfo.getName())); selectedAudioIndex = ListHelper.getDefaultAudioFormat(getContext(), currentInfo.getAudioStreams()); + selectedSubtitleIndex = getSubtitleIndexBy(subtitleStreamsAdapter.getAll()); + streamsSpinner = view.findViewById(R.id.quality_spinner); streamsSpinner.setOnItemSelectedListener(this); @@ -154,14 +210,18 @@ public class DownloadDialog extends DialogFragment implements RadioGroup.OnCheck initToolbar(view.findViewById(R.id.toolbar)); setupDownloadOptions(); - int def = 3; - threadsCountTextView.setText(String.valueOf(def)); - threadsSeekBar.setProgress(def - 1); + prefs = PreferenceManager.getDefaultSharedPreferences(getContext()); + + int threads = prefs.getInt(getString(R.string.default_download_threads), 3); + threadsCountTextView.setText(String.valueOf(threads)); + threadsSeekBar.setProgress(threads - 1); threadsSeekBar.setOnSeekBarChangeListener(new SeekBar.OnSeekBarChangeListener() { @Override public void onProgressChanged(SeekBar seekbar, int progress, boolean fromUser) { - threadsCountTextView.setText(String.valueOf(progress + 1)); + progress++; + prefs.edit().putInt(getString(R.string.default_download_threads), progress).apply(); + threadsCountTextView.setText(String.valueOf(progress)); } @Override @@ -189,6 +249,11 @@ public class DownloadDialog extends DialogFragment implements RadioGroup.OnCheck setupAudioSpinner(); } })); + disposables.add(StreamSizeWrapper.fetchSizeForWrapper(wrappedSubtitleStreams).subscribe(result -> { + if (radioVideoAudioGroup.getCheckedRadioButtonId() == R.id.subtitle_button) { + setupSubtitleSpinner(); + } + })); } @Override @@ -216,7 +281,7 @@ public class DownloadDialog extends DialogFragment implements RadioGroup.OnCheck toolbar.setOnMenuItemClickListener(item -> { if (item.getItemId() == R.id.okay) { - downloadSelected(); + prepareSelectedDownload(); return true; } return false; @@ -239,13 +304,24 @@ public class DownloadDialog extends DialogFragment implements RadioGroup.OnCheck setRadioButtonsState(true); } + private void setupSubtitleSpinner() { + if (getContext() == null) return; + + streamsSpinner.setAdapter(subtitleStreamsAdapter); + streamsSpinner.setSelection(selectedSubtitleIndex); + setRadioButtonsState(true); + } + /*////////////////////////////////////////////////////////////////////////// // Radio group Video&Audio options - Listener //////////////////////////////////////////////////////////////////////////*/ @Override public void onCheckedChanged(RadioGroup group, @IdRes int checkedId) { - if (DEBUG) Log.d(TAG, "onCheckedChanged() called with: group = [" + group + "], checkedId = [" + checkedId + "]"); + if (DEBUG) + Log.d(TAG, "onCheckedChanged() called with: group = [" + group + "], checkedId = [" + checkedId + "]"); + boolean flag = true; + switch (checkedId) { case R.id.audio_button: setupAudioSpinner(); @@ -253,7 +329,13 @@ public class DownloadDialog extends DialogFragment implements RadioGroup.OnCheck case R.id.video_button: setupVideoSpinner(); break; + case R.id.subtitle_button: + setupSubtitleSpinner(); + flag = false; + break; } + + threadsSeekBar.setEnabled(flag); } /*////////////////////////////////////////////////////////////////////////// @@ -262,7 +344,8 @@ public class DownloadDialog extends DialogFragment implements RadioGroup.OnCheck @Override public void onItemSelected(AdapterView parent, View view, int position, long id) { - if (DEBUG) Log.d(TAG, "onItemSelected() called with: parent = [" + parent + "], view = [" + view + "], position = [" + position + "], id = [" + id + "]"); + if (DEBUG) + Log.d(TAG, "onItemSelected() called with: parent = [" + parent + "], view = [" + view + "], position = [" + position + "], id = [" + id + "]"); switch (radioVideoAudioGroup.getCheckedRadioButtonId()) { case R.id.audio_button: selectedAudioIndex = position; @@ -270,6 +353,9 @@ public class DownloadDialog extends DialogFragment implements RadioGroup.OnCheck case R.id.video_button: selectedVideoIndex = position; break; + case R.id.subtitle_button: + selectedSubtitleIndex = position; + break; } } @@ -286,11 +372,14 @@ public class DownloadDialog extends DialogFragment implements RadioGroup.OnCheck final RadioButton audioButton = radioVideoAudioGroup.findViewById(R.id.audio_button); final RadioButton videoButton = radioVideoAudioGroup.findViewById(R.id.video_button); + final RadioButton subtitleButton = radioVideoAudioGroup.findViewById(R.id.subtitle_button); final boolean isVideoStreamsAvailable = videoStreamsAdapter.getCount() > 0; final boolean isAudioStreamsAvailable = audioStreamsAdapter.getCount() > 0; + final boolean isSubtitleStreamsAvailable = subtitleStreamsAdapter.getCount() > 0; audioButton.setVisibility(isAudioStreamsAvailable ? View.VISIBLE : View.GONE); videoButton.setVisibility(isVideoStreamsAvailable ? View.VISIBLE : View.GONE); + subtitleButton.setVisibility(isSubtitleStreamsAvailable ? View.VISIBLE : View.GONE); if (isVideoStreamsAvailable) { videoButton.setChecked(true); @@ -298,6 +387,9 @@ public class DownloadDialog extends DialogFragment implements RadioGroup.OnCheck } else if (isAudioStreamsAvailable) { audioButton.setChecked(true); setupAudioSpinner(); + } else if (isSubtitleStreamsAvailable) { + subtitleButton.setChecked(true); + setupSubtitleSpinner(); } else { Toast.makeText(getContext(), R.string.no_streams_available_download, Toast.LENGTH_SHORT).show(); getDialog().dismiss(); @@ -307,28 +399,144 @@ public class DownloadDialog extends DialogFragment implements RadioGroup.OnCheck private void setRadioButtonsState(boolean enabled) { radioVideoAudioGroup.findViewById(R.id.audio_button).setEnabled(enabled); radioVideoAudioGroup.findViewById(R.id.video_button).setEnabled(enabled); + radioVideoAudioGroup.findViewById(R.id.subtitle_button).setEnabled(enabled); } - private void downloadSelected() { - Stream stream; - String location; + private int getSubtitleIndexBy(List streams) { + Localization loc = NewPipe.getPreferredLocalization(); - String fileName = nameEditText.getText().toString().trim(); - if (fileName.isEmpty()) fileName = FilenameUtils.createFilename(getContext(), currentInfo.getName()); - - boolean isAudio = radioVideoAudioGroup.getCheckedRadioButtonId() == R.id.audio_button; - if (isAudio) { - stream = audioStreamsAdapter.getItem(selectedAudioIndex); - location = NewPipeSettings.getAudioDownloadPath(getContext()); - } else { - stream = videoStreamsAdapter.getItem(selectedVideoIndex); - location = NewPipeSettings.getVideoDownloadPath(getContext()); + for (int i = 0; i < streams.size(); i++) { + Locale streamLocale = streams.get(i).getLocale(); + String tag = streamLocale.getLanguage().concat("-").concat(streamLocale.getCountry()); + if (tag.equalsIgnoreCase(loc.getLanguage())) { + return i; + } } - String url = stream.getUrl(); - fileName += "." + stream.getFormat().getSuffix(); + // fallback + // 1st loop match country & language + // 2nd loop match language only + int index = loc.getLanguage().indexOf("-"); + String lang = index > 0 ? loc.getLanguage().substring(0, index) : loc.getLanguage(); + + for (int j = 0; j < 2; j++) { + for (int i = 0; i < streams.size(); i++) { + Locale streamLocale = streams.get(i).getLocale(); + + if (streamLocale.getLanguage().equalsIgnoreCase(lang)) { + if (j > 0 || streamLocale.getCountry().equalsIgnoreCase(loc.getCountry())) { + return i; + } + } + } + } + + return 0; + } + + private void prepareSelectedDownload() { + final Context context = getContext(); + Stream stream; + String location; + char kind; + + String fileName = nameEditText.getText().toString().trim(); + if (fileName.isEmpty()) + fileName = FilenameUtils.createFilename(context, currentInfo.getName()); + + switch (radioVideoAudioGroup.getCheckedRadioButtonId()) { + case R.id.audio_button: + stream = audioStreamsAdapter.getItem(selectedAudioIndex); + location = NewPipeSettings.getAudioDownloadPath(context); + kind = 'a'; + break; + case R.id.video_button: + stream = videoStreamsAdapter.getItem(selectedVideoIndex); + location = NewPipeSettings.getVideoDownloadPath(context); + kind = 'v'; + break; + case R.id.subtitle_button: + stream = subtitleStreamsAdapter.getItem(selectedSubtitleIndex); + location = NewPipeSettings.getVideoDownloadPath(context);// assume that subtitle & video go together + kind = 's'; + break; + default: + return; + } + + int threads; + + if (radioVideoAudioGroup.getCheckedRadioButtonId() == R.id.subtitle_button) { + threads = 1;// use unique thread for subtitles due small file size + fileName += ".srt";// final subtitle format + } else { + threads = threadsSeekBar.getProgress() + 1; + fileName += "." + stream.getFormat().getSuffix(); + } + + final String finalFileName = fileName; + + DownloadManagerService.checkForRunningMission(context, location, fileName, (listed, finished) -> { + // should be safe run the following code without "getActivity().runOnUiThread()" + if (listed) { + AlertDialog.Builder builder = new AlertDialog.Builder(context); + builder.setTitle(R.string.download_dialog_title) + .setMessage(finished ? R.string.overwrite_warning : R.string.download_already_running) + .setPositiveButton( + finished ? R.string.overwrite : R.string.generate_unique_name, + (dialog, which) -> downloadSelected(context, stream, location, finalFileName, kind, threads) + ) + .setNegativeButton(android.R.string.cancel, (dialog, which) -> { + dialog.cancel(); + }) + .create() + .show(); + } else { + downloadSelected(context, stream, location, finalFileName, kind, threads); + } + }); + } + + private void downloadSelected(Context context, Stream selectedStream, String location, String fileName, char kind, int threads) { + String[] urls; + String psName = null; + String[] psArgs = null; + String secondaryStreamUrl = null; + long nearLength = 0; + + if (selectedStream instanceof VideoStream) { + SecondaryStreamHelper secondaryStream = videoStreamsAdapter + .getAllSecondary() + .get(wrappedVideoStreams.getStreamsList().indexOf(selectedStream)); + + if (secondaryStream != null) { + secondaryStreamUrl = secondaryStream.getStream().getUrl(); + psName = selectedStream.getFormat() == MediaFormat.MPEG_4 ? Postprocessing.ALGORITHM_MP4_DASH_MUXER : Postprocessing.ALGORITHM_WEBM_MUXER; + psArgs = null; + long videoSize = wrappedVideoStreams.getSizeInBytes((VideoStream) selectedStream); + + // set nearLength, only, if both sizes are fetched or known. this probably does not work on weak internet connections + if (secondaryStream.getSizeInBytes() > 0 && videoSize > 0) { + nearLength = secondaryStream.getSizeInBytes() + videoSize; + } + } + } else if ((selectedStream instanceof SubtitlesStream) && selectedStream.getFormat() == MediaFormat.TTML) { + psName = Postprocessing.ALGORITHM_TTML_CONVERTER; + psArgs = new String[]{ + selectedStream.getFormat().getSuffix(), + "false",// ignore empty frames + "false",// detect youtube duplicate lines + }; + } + + if (secondaryStreamUrl == null) { + urls = new String[]{selectedStream.getUrl()}; + } else { + urls = new String[]{selectedStream.getUrl(), secondaryStreamUrl}; + } + + DownloadManagerService.startMission(context, urls, location, fileName, kind, threads, currentInfo.getUrl(), psName, psArgs, nearLength); - DownloadManagerService.startMission(getContext(), url, location, fileName, isAudio, threadsSeekBar.getProgress() + 1); getDialog().dismiss(); } } diff --git a/app/src/main/java/org/schabi/newpipe/fragments/detail/VideoDetailFragment.java b/app/src/main/java/org/schabi/newpipe/fragments/detail/VideoDetailFragment.java index f5675dcb2..edca200d7 100644 --- a/app/src/main/java/org/schabi/newpipe/fragments/detail/VideoDetailFragment.java +++ b/app/src/main/java/org/schabi/newpipe/fragments/detail/VideoDetailFragment.java @@ -63,6 +63,7 @@ import org.schabi.newpipe.extractor.stream.Stream; import org.schabi.newpipe.extractor.stream.StreamInfo; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamType; +import org.schabi.newpipe.extractor.stream.SubtitlesStream; import org.schabi.newpipe.extractor.stream.VideoStream; import org.schabi.newpipe.fragments.BackPressable; import org.schabi.newpipe.fragments.BaseStateFragment; @@ -370,14 +371,14 @@ public class VideoDetailFragment Log.w(TAG, "Can't open channel because we got no channel URL"); } else { try { - NavigationHelper.openChannelFragment( - getFragmentManager(), - currentInfo.getServiceId(), - currentInfo.getUploaderUrl(), - currentInfo.getUploaderName()); + NavigationHelper.openChannelFragment( + getFragmentManager(), + currentInfo.getServiceId(), + currentInfo.getUploaderUrl(), + currentInfo.getUploaderName()); } catch (Exception e) { ErrorActivity.reportUiError((AppCompatActivity) getActivity(), e); - } + } } break; case R.id.detail_thumbnail_root_layout: @@ -744,7 +745,7 @@ public class VideoDetailFragment sortedVideoStreams = ListHelper.getSortedStreamVideosList(activity, info.getVideoStreams(), info.getVideoOnlyStreams(), false); selectedVideoStreamIndex = ListHelper.getDefaultResolutionIndex(activity, sortedVideoStreams); - final StreamItemAdapter streamsAdapter = new StreamItemAdapter<>(activity, new StreamSizeWrapper<>(sortedVideoStreams), isExternalPlayerEnabled); + final StreamItemAdapter streamsAdapter = new StreamItemAdapter<>(activity, new StreamSizeWrapper<>(sortedVideoStreams, activity), isExternalPlayerEnabled); spinnerToolbar.setAdapter(streamsAdapter); spinnerToolbar.setSelection(selectedVideoStreamIndex); spinnerToolbar.setOnItemSelectedListener(new AdapterView.OnItemSelectedListener() { @@ -1264,6 +1265,7 @@ public class VideoDetailFragment downloadDialog.setVideoStreams(sortedVideoStreams); downloadDialog.setAudioStreams(currentInfo.getAudioStreams()); downloadDialog.setSelectedVideoStream(selectedVideoStreamIndex); + downloadDialog.setSubtitleStreams(currentInfo.getSubtitles()); downloadDialog.show(activity.getSupportFragmentManager(), "downloadDialog"); } catch (Exception e) { @@ -1321,4 +1323,4 @@ public class VideoDetailFragment relatedStreamRootLayout.setVisibility(visibility); } } -} \ No newline at end of file +} diff --git a/app/src/main/java/org/schabi/newpipe/player/helper/PlayerHelper.java b/app/src/main/java/org/schabi/newpipe/player/helper/PlayerHelper.java index 33a29296c..19b728b3a 100644 --- a/app/src/main/java/org/schabi/newpipe/player/helper/PlayerHelper.java +++ b/app/src/main/java/org/schabi/newpipe/player/helper/PlayerHelper.java @@ -19,11 +19,11 @@ import com.google.android.exoplayer2.util.MimeTypes; import org.schabi.newpipe.R; import org.schabi.newpipe.extractor.InfoItem; -import org.schabi.newpipe.extractor.Subtitles; +import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.stream.AudioStream; import org.schabi.newpipe.extractor.stream.StreamInfo; import org.schabi.newpipe.extractor.stream.StreamInfoItem; -import org.schabi.newpipe.extractor.stream.SubtitlesFormat; +import org.schabi.newpipe.extractor.stream.SubtitlesStream; import org.schabi.newpipe.extractor.stream.VideoStream; import org.schabi.newpipe.player.playqueue.PlayQueue; import org.schabi.newpipe.player.playqueue.PlayQueueItem; @@ -87,7 +87,7 @@ public class PlayerHelper { return pitchFormatter.format(pitch); } - public static String mimeTypesOf(final SubtitlesFormat format) { + public static String subtitleMimeTypesOf(final MediaFormat format) { switch (format) { case VTT: return MimeTypes.TEXT_VTT; case TTML: return MimeTypes.APPLICATION_TTML; @@ -97,8 +97,8 @@ public class PlayerHelper { @NonNull public static String captionLanguageOf(@NonNull final Context context, - @NonNull final Subtitles subtitles) { - final String displayName = subtitles.getLocale().getDisplayName(subtitles.getLocale()); + @NonNull final SubtitlesStream subtitles) { + final String displayName = subtitles.getDisplayLanguageName(); return displayName + (subtitles.isAutoGenerated() ? " (" + context.getString(R.string.caption_auto_generated)+ ")" : ""); } diff --git a/app/src/main/java/org/schabi/newpipe/player/resolver/VideoPlaybackResolver.java b/app/src/main/java/org/schabi/newpipe/player/resolver/VideoPlaybackResolver.java index 8f91f4886..ad2b79523 100644 --- a/app/src/main/java/org/schabi/newpipe/player/resolver/VideoPlaybackResolver.java +++ b/app/src/main/java/org/schabi/newpipe/player/resolver/VideoPlaybackResolver.java @@ -10,7 +10,7 @@ import com.google.android.exoplayer2.source.MediaSource; import com.google.android.exoplayer2.source.MergingMediaSource; import org.schabi.newpipe.extractor.MediaFormat; -import org.schabi.newpipe.extractor.Subtitles; +import org.schabi.newpipe.extractor.stream.SubtitlesStream; import org.schabi.newpipe.extractor.stream.AudioStream; import org.schabi.newpipe.extractor.stream.StreamInfo; import org.schabi.newpipe.extractor.stream.VideoStream; @@ -93,8 +93,8 @@ public class VideoPlaybackResolver implements PlaybackResolver { // Below are auxiliary media sources // Create subtitle sources - for (final Subtitles subtitle : info.getSubtitles()) { - final String mimeType = PlayerHelper.mimeTypesOf(subtitle.getFileType()); + for (final SubtitlesStream subtitle : info.getSubtitles()) { + final String mimeType = PlayerHelper.subtitleMimeTypesOf(subtitle.getFormat()); if (mimeType == null) continue; final Format textFormat = Format.createTextSampleFormat(null, mimeType, diff --git a/app/src/main/java/org/schabi/newpipe/streams/DataReader.java b/app/src/main/java/org/schabi/newpipe/streams/DataReader.java new file mode 100644 index 000000000..d0e946eb7 --- /dev/null +++ b/app/src/main/java/org/schabi/newpipe/streams/DataReader.java @@ -0,0 +1,103 @@ +package org.schabi.newpipe.streams; + +import java.io.EOFException; +import java.io.IOException; + +import org.schabi.newpipe.streams.io.SharpStream; + +/** + * @author kapodamy + */ +public class DataReader { + + public final static int SHORT_SIZE = 2; + public final static int LONG_SIZE = 8; + public final static int INTEGER_SIZE = 4; + public final static int FLOAT_SIZE = 4; + + private long pos; + public final SharpStream stream; + private final boolean rewind; + + public DataReader(SharpStream stream) { + this.rewind = stream.canRewind(); + this.stream = stream; + this.pos = 0L; + } + + public long position() { + return pos; + } + + public final int readInt() throws IOException { + primitiveRead(INTEGER_SIZE); + return primitive[0] << 24 | primitive[1] << 16 | primitive[2] << 8 | primitive[3]; + } + + public final int read() throws IOException { + int value = stream.read(); + if (value == -1) { + throw new EOFException(); + } + + pos++; + return value; + } + + public final long skipBytes(long amount) throws IOException { + amount = stream.skip(amount); + pos += amount; + return amount; + } + + public final long readLong() throws IOException { + primitiveRead(LONG_SIZE); + long high = primitive[0] << 24 | primitive[1] << 16 | primitive[2] << 8 | primitive[3]; + long low = primitive[4] << 24 | primitive[5] << 16 | primitive[6] << 8 | primitive[7]; + return high << 32 | low; + } + + public final short readShort() throws IOException { + primitiveRead(SHORT_SIZE); + return (short) (primitive[0] << 8 | primitive[1]); + } + + public final int read(byte[] buffer) throws IOException { + return read(buffer, 0, buffer.length); + } + + public final int read(byte[] buffer, int offset, int count) throws IOException { + int res = stream.read(buffer, offset, count); + pos += res; + + return res; + } + + public final boolean available() { + return stream.available() > 0; + } + + public void rewind() throws IOException { + stream.rewind(); + pos = 0; + } + + public boolean canRewind() { + return rewind; + } + + private short[] primitive = new short[LONG_SIZE]; + + private void primitiveRead(int amount) throws IOException { + byte[] buffer = new byte[amount]; + int read = stream.read(buffer, 0, amount); + pos += read; + if (read != amount) { + throw new EOFException("Truncated data, missing " + String.valueOf(amount - read) + " bytes"); + } + + for (int i = 0; i < buffer.length; i++) { + primitive[i] = (short) (buffer[i] & 0xFF);// the "byte" datatype is signed and is very annoying + } + } +} diff --git a/app/src/main/java/org/schabi/newpipe/streams/Mp4DashReader.java b/app/src/main/java/org/schabi/newpipe/streams/Mp4DashReader.java new file mode 100644 index 000000000..271929d47 --- /dev/null +++ b/app/src/main/java/org/schabi/newpipe/streams/Mp4DashReader.java @@ -0,0 +1,817 @@ +package org.schabi.newpipe.streams; + +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStream; +import java.io.UnsupportedEncodingException; + +import java.nio.ByteBuffer; + +import java.util.ArrayList; +import java.util.NoSuchElementException; + +import org.schabi.newpipe.streams.io.SharpStream; + +/** + * @author kapodamy + */ +public class Mp4DashReader { + + // + private static final int ATOM_MOOF = 0x6D6F6F66; + private static final int ATOM_MFHD = 0x6D666864; + private static final int ATOM_TRAF = 0x74726166; + private static final int ATOM_TFHD = 0x74666864; + private static final int ATOM_TFDT = 0x74666474; + private static final int ATOM_TRUN = 0x7472756E; + private static final int ATOM_MDIA = 0x6D646961; + private static final int ATOM_FTYP = 0x66747970; + private static final int ATOM_SIDX = 0x73696478; + private static final int ATOM_MOOV = 0x6D6F6F76; + private static final int ATOM_MDAT = 0x6D646174; + private static final int ATOM_MVHD = 0x6D766864; + private static final int ATOM_TRAK = 0x7472616B; + private static final int ATOM_MVEX = 0x6D766578; + private static final int ATOM_TREX = 0x74726578; + private static final int ATOM_TKHD = 0x746B6864; + private static final int ATOM_MFRA = 0x6D667261; + private static final int ATOM_TFRA = 0x74667261; + private static final int ATOM_MDHD = 0x6D646864; + private static final int BRAND_DASH = 0x64617368; + // + + private final DataReader stream; + + private Mp4Track[] tracks = null; + + private Box box; + private Moof moof; + + private boolean chunkZero = false; + + private int selectedTrack = -1; + + public enum TrackKind { + Audio, Video, Other + } + + public Mp4DashReader(SharpStream source) { + this.stream = new DataReader(source); + } + + public void parse() throws IOException, NoSuchElementException { + if (selectedTrack > -1) { + return; + } + + box = readBox(ATOM_FTYP); + if (parse_ftyp() != BRAND_DASH) { + throw new NoSuchElementException("Main Brand is not dash"); + } + + Moov moov = null; + int i; + + while (box.type != ATOM_MOOF) { + ensure(box); + box = readBox(); + + switch (box.type) { + case ATOM_MOOV: + moov = parse_moov(box); + break; + case ATOM_SIDX: + break; + case ATOM_MFRA: + break; + case ATOM_MDAT: + throw new IOException("Expected moof, found mdat"); + } + } + + if (moov == null) { + throw new IOException("The provided Mp4 doesn't have the 'moov' box"); + } + + tracks = new Mp4Track[moov.trak.length]; + + for (i = 0; i < tracks.length; i++) { + tracks[i] = new Mp4Track(); + tracks[i].trak = moov.trak[i]; + + if (moov.mvex_trex != null) { + for (Trex mvex_trex : moov.mvex_trex) { + if (tracks[i].trak.tkhd.trackId == mvex_trex.trackId) { + tracks[i].trex = mvex_trex; + } + } + } + + if (moov.trak[i].tkhd.bHeight == 0 && moov.trak[i].tkhd.bWidth == 0) { + tracks[i].kind = moov.trak[i].tkhd.bVolume == 0 ? TrackKind.Other : TrackKind.Audio; + } else { + tracks[i].kind = TrackKind.Video; + } + } + } + + public Mp4Track selectTrack(int index) { + selectedTrack = index; + return tracks[index]; + } + + /** + * Count all fragments present. This operation requires a seekable stream + * + * @return list with a basic info + * @throws IOException if the source stream is not seekeable + */ + public int getFragmentsCount() throws IOException { + if (selectedTrack < 0) { + throw new IllegalStateException("track no selected"); + } + if (!stream.canRewind()) { + throw new IOException("The provided stream doesn't allow seek"); + } + + Box tmp; + int count = 0; + long orig_offset = stream.position(); + + if (box.type == ATOM_MOOF) { + tmp = box; + } else { + ensure(box); + tmp = readBox(); + } + + do { + if (tmp.type == ATOM_MOOF) { + ensure(readBox(ATOM_MFHD)); + Box traf; + while ((traf = untilBox(tmp, ATOM_TRAF)) != null) { + Box tfhd = readBox(ATOM_TFHD); + if (parse_tfhd(tracks[selectedTrack].trak.tkhd.trackId) != null) { + count++; + break; + } + ensure(tfhd); + ensure(traf); + } + } + ensure(tmp); + } while (stream.available() && (tmp = readBox()) != null); + + stream.rewind(); + stream.skipBytes((int) orig_offset); + + return count; + } + + public Mp4Track[] getAvailableTracks() { + return tracks; + } + + public Mp4TrackChunk getNextChunk() throws IOException { + Mp4Track track = tracks[selectedTrack]; + + while (stream.available()) { + + if (chunkZero) { + ensure(box); + if (!stream.available()) { + break; + } + box = readBox(); + } else { + chunkZero = true; + } + + switch (box.type) { + case ATOM_MOOF: + if (moof != null) { + throw new IOException("moof found without mdat"); + } + + moof = parse_moof(box, track.trak.tkhd.trackId); + + if (moof.traf != null) { + + if (hasFlag(moof.traf.trun.bFlags, 0x0001)) { + moof.traf.trun.dataOffset -= box.size + 8; + if (moof.traf.trun.dataOffset < 0) { + throw new IOException("trun box has wrong data offset, points outside of concurrent mdat box"); + } + } + + if (moof.traf.trun.chunkSize < 1) { + if (hasFlag(moof.traf.tfhd.bFlags, 0x10)) { + moof.traf.trun.chunkSize = moof.traf.tfhd.defaultSampleSize * moof.traf.trun.entryCount; + } else { + moof.traf.trun.chunkSize = box.size - 8; + } + } + if (!hasFlag(moof.traf.trun.bFlags, 0x900) && moof.traf.trun.chunkDuration == 0) { + if (hasFlag(moof.traf.tfhd.bFlags, 0x20)) { + moof.traf.trun.chunkDuration = moof.traf.tfhd.defaultSampleDuration * moof.traf.trun.entryCount; + } + } + } + break; + case ATOM_MDAT: + if (moof == null) { + throw new IOException("mdat found without moof"); + } + + if (moof.traf == null) { + moof = null; + continue;// find another chunk + } + + Mp4TrackChunk chunk = new Mp4TrackChunk(); + chunk.moof = moof; + chunk.data = new TrackDataChunk(stream, moof.traf.trun.chunkSize); + moof = null; + + stream.skipBytes(chunk.moof.traf.trun.dataOffset); + return chunk; + default: + } + } + + return null; + } + + // + private long readUint() throws IOException { + return stream.readInt() & 0xffffffffL; + } + + public static boolean hasFlag(int flags, int mask) { + return (flags & mask) == mask; + } + + private String boxName(Box ref) { + return boxName(ref.type); + } + + private String boxName(int type) { + try { + return new String(ByteBuffer.allocate(4).putInt(type).array(), "UTF-8"); + } catch (UnsupportedEncodingException e) { + return "0x" + Integer.toHexString(type); + } + } + + private Box readBox() throws IOException { + Box b = new Box(); + b.offset = stream.position(); + b.size = stream.readInt(); + b.type = stream.readInt(); + + return b; + } + + private Box readBox(int expected) throws IOException { + Box b = readBox(); + if (b.type != expected) { + throw new NoSuchElementException("expected " + boxName(expected) + " found " + boxName(b)); + } + return b; + } + + private void ensure(Box ref) throws IOException { + long skip = ref.offset + ref.size - stream.position(); + + if (skip == 0) { + return; + } else if (skip < 0) { + throw new EOFException(String.format( + "parser go beyond limits of the box. type=%s offset=%s size=%s position=%s", + boxName(ref), ref.offset, ref.size, stream.position() + )); + } + + stream.skipBytes((int) skip); + } + + private Box untilBox(Box ref, int... expected) throws IOException { + Box b; + while (stream.position() < (ref.offset + ref.size)) { + b = readBox(); + for (int type : expected) { + if (b.type == type) { + return b; + } + } + ensure(b); + } + + return null; + } + + // + + // + + private Moof parse_moof(Box ref, int trackId) throws IOException { + Moof obj = new Moof(); + + Box b = readBox(ATOM_MFHD); + obj.mfhd_SequenceNumber = parse_mfhd(); + ensure(b); + + while ((b = untilBox(ref, ATOM_TRAF)) != null) { + obj.traf = parse_traf(b, trackId); + ensure(b); + + if (obj.traf != null) { + return obj; + } + } + + return obj; + } + + private int parse_mfhd() throws IOException { + // version + // flags + stream.skipBytes(4); + + return stream.readInt(); + } + + private Traf parse_traf(Box ref, int trackId) throws IOException { + Traf traf = new Traf(); + + Box b = readBox(ATOM_TFHD); + traf.tfhd = parse_tfhd(trackId); + ensure(b); + + if (traf.tfhd == null) { + return null; + } + + b = untilBox(ref, ATOM_TRUN, ATOM_TFDT); + + if (b.type == ATOM_TFDT) { + traf.tfdt = parse_tfdt(); + ensure(b); + b = readBox(ATOM_TRUN); + } + + traf.trun = parse_trun(); + ensure(b); + + return traf; + } + + private Tfhd parse_tfhd(int trackId) throws IOException { + Tfhd obj = new Tfhd(); + + obj.bFlags = stream.readInt(); + obj.trackId = stream.readInt(); + + if (trackId != -1 && obj.trackId != trackId) { + return null; + } + + if (hasFlag(obj.bFlags, 0x01)) { + stream.skipBytes(8); + } + if (hasFlag(obj.bFlags, 0x02)) { + stream.skipBytes(4); + } + if (hasFlag(obj.bFlags, 0x08)) { + obj.defaultSampleDuration = stream.readInt(); + } + if (hasFlag(obj.bFlags, 0x10)) { + obj.defaultSampleSize = stream.readInt(); + } + if (hasFlag(obj.bFlags, 0x20)) { + obj.defaultSampleFlags = stream.readInt(); + } + + return obj; + } + + private long parse_tfdt() throws IOException { + int version = stream.read(); + stream.skipBytes(3);// flags + return version == 0 ? readUint() : stream.readLong(); + } + + private Trun parse_trun() throws IOException { + Trun obj = new Trun(); + obj.bFlags = stream.readInt(); + obj.entryCount = stream.readInt();// unsigned int + + obj.entries_rowSize = 0; + if (hasFlag(obj.bFlags, 0x0100)) { + obj.entries_rowSize += 4; + } + if (hasFlag(obj.bFlags, 0x0200)) { + obj.entries_rowSize += 4; + } + if (hasFlag(obj.bFlags, 0x0400)) { + obj.entries_rowSize += 4; + } + if (hasFlag(obj.bFlags, 0x0800)) { + obj.entries_rowSize += 4; + } + obj.bEntries = new byte[obj.entries_rowSize * obj.entryCount]; + + if (hasFlag(obj.bFlags, 0x0001)) { + obj.dataOffset = stream.readInt(); + } + if (hasFlag(obj.bFlags, 0x0004)) { + obj.bFirstSampleFlags = stream.readInt(); + } + + stream.read(obj.bEntries); + + for (int i = 0; i < obj.entryCount; i++) { + TrunEntry entry = obj.getEntry(i); + if (hasFlag(obj.bFlags, 0x0100)) { + obj.chunkDuration += entry.sampleDuration; + } + if (hasFlag(obj.bFlags, 0x0200)) { + obj.chunkSize += entry.sampleSize; + } + if (hasFlag(obj.bFlags, 0x0800)) { + if (!hasFlag(obj.bFlags, 0x0100)) { + obj.chunkDuration += entry.sampleCompositionTimeOffset; + } + } + } + + return obj; + } + + private int parse_ftyp() throws IOException { + int brand = stream.readInt(); + stream.skipBytes(4);// minor version + + return brand; + } + + private Mvhd parse_mvhd() throws IOException { + int version = stream.read(); + stream.skipBytes(3);// flags + + // creation entries_time + // modification entries_time + stream.skipBytes(2 * (version == 0 ? 4 : 8)); + + Mvhd obj = new Mvhd(); + obj.timeScale = readUint(); + + // chunkDuration + stream.skipBytes(version == 0 ? 4 : 8); + + // rate + // volume + // reserved + // matrix array + // predefined + stream.skipBytes(76); + + obj.nextTrackId = readUint(); + + return obj; + } + + private Tkhd parse_tkhd() throws IOException { + int version = stream.read(); + + Tkhd obj = new Tkhd(); + + // flags + // creation entries_time + // modification entries_time + stream.skipBytes(3 + (2 * (version == 0 ? 4 : 8))); + + obj.trackId = stream.readInt(); + + stream.skipBytes(4);// reserved + + obj.duration = version == 0 ? readUint() : stream.readLong(); + + stream.skipBytes(2 * 4);// reserved + + obj.bLayer = stream.readShort(); + obj.bAlternateGroup = stream.readShort(); + obj.bVolume = stream.readShort(); + + stream.skipBytes(2);// reserved + + obj.matrix = new byte[9 * 4]; + stream.read(obj.matrix); + + obj.bWidth = stream.readInt(); + obj.bHeight = stream.readInt(); + + return obj; + } + + private Trak parse_trak(Box ref) throws IOException { + Trak trak = new Trak(); + + Box b = readBox(ATOM_TKHD); + trak.tkhd = parse_tkhd(); + ensure(b); + + b = untilBox(ref, ATOM_MDIA); + trak.mdia = new byte[b.size]; + + ByteBuffer buffer = ByteBuffer.wrap(trak.mdia); + buffer.putInt(b.size); + buffer.putInt(ATOM_MDIA); + stream.read(trak.mdia, 8, b.size - 8); + + trak.mdia_mdhd_timeScale = parse_mdia(buffer); + + return trak; + } + + private int parse_mdia(ByteBuffer data) { + while (data.hasRemaining()) { + int end = data.position() + data.getInt(); + if (data.getInt() == ATOM_MDHD) { + byte version = data.get(); + data.position(data.position() + 3 + ((version == 0 ? 4 : 8) * 2)); + return data.getInt(); + } + + data.position(end); + } + + return 0;// this NEVER should happen + } + + private Moov parse_moov(Box ref) throws IOException { + Box b = readBox(ATOM_MVHD); + Moov moov = new Moov(); + moov.mvhd = parse_mvhd(); + ensure(b); + + ArrayList tmp = new ArrayList<>((int) moov.mvhd.nextTrackId); + while ((b = untilBox(ref, ATOM_TRAK, ATOM_MVEX)) != null) { + + switch (b.type) { + case ATOM_TRAK: + tmp.add(parse_trak(b)); + break; + case ATOM_MVEX: + moov.mvex_trex = parse_mvex(b, (int) moov.mvhd.nextTrackId); + break; + } + + ensure(b); + } + + moov.trak = tmp.toArray(new Trak[tmp.size()]); + + return moov; + } + + private Trex[] parse_mvex(Box ref, int possibleTrackCount) throws IOException { + ArrayList tmp = new ArrayList<>(possibleTrackCount); + + Box b; + while ((b = untilBox(ref, ATOM_TREX)) != null) { + tmp.add(parse_trex()); + ensure(b); + } + + return tmp.toArray(new Trex[tmp.size()]); + } + + private Trex parse_trex() throws IOException { + // version + // flags + stream.skipBytes(4); + + Trex obj = new Trex(); + obj.trackId = stream.readInt(); + obj.defaultSampleDescriptionIndex = stream.readInt(); + obj.defaultSampleDuration = stream.readInt(); + obj.defaultSampleSize = stream.readInt(); + obj.defaultSampleFlags = stream.readInt(); + + return obj; + } + + private Tfra parse_tfra() throws IOException { + int version = stream.read(); + + stream.skipBytes(3);// flags + + Tfra tfra = new Tfra(); + tfra.trackId = stream.readInt(); + + stream.skipBytes(3);// reserved + int bFlags = stream.read(); + int size_tts = ((bFlags >> 4) & 3) + ((bFlags >> 2) & 3) + (bFlags & 3); + + tfra.entries_time = new int[stream.readInt()]; + + for (int i = 0; i < tfra.entries_time.length; i++) { + tfra.entries_time[i] = version == 0 ? stream.readInt() : (int) stream.readLong(); + stream.skipBytes(size_tts + (version == 0 ? 4 : 8)); + } + + return tfra; + } + + private Sidx parse_sidx() throws IOException { + int version = stream.read(); + + stream.skipBytes(3);// flags + + Sidx obj = new Sidx(); + obj.referenceId = stream.readInt(); + obj.timescale = stream.readInt(); + + // earliest presentation entries_time + // first offset + // reserved + stream.skipBytes((2 * (version == 0 ? 4 : 8)) + 2); + + obj.entries_subsegmentDuration = new int[stream.readShort()]; + + for (int i = 0; i < obj.entries_subsegmentDuration.length; i++) { + // reference type + // referenced size + stream.skipBytes(4); + obj.entries_subsegmentDuration[i] = stream.readInt();// unsigned int + + // starts with SAP + // SAP type + // SAP delta entries_time + stream.skipBytes(4); + } + + return obj; + } + + private Tfra[] parse_mfra(Box ref, int trackCount) throws IOException { + ArrayList tmp = new ArrayList<>(trackCount); + long limit = ref.offset + ref.size; + + while (stream.position() < limit) { + box = readBox(); + + if (box.type == ATOM_TFRA) { + tmp.add(parse_tfra()); + } + + ensure(box); + } + + return tmp.toArray(new Tfra[tmp.size()]); + } + + // + + // + class Box { + + int type; + long offset; + int size; + } + + class Sidx { + + int timescale; + int referenceId; + int[] entries_subsegmentDuration; + } + + public class Moof { + + int mfhd_SequenceNumber; + public Traf traf; + } + + public class Traf { + + public Tfhd tfhd; + long tfdt; + public Trun trun; + } + + public class Tfhd { + + int bFlags; + public int trackId; + int defaultSampleDuration; + int defaultSampleSize; + int defaultSampleFlags; + } + + public class TrunEntry { + + public int sampleDuration; + public int sampleSize; + public int sampleFlags; + public int sampleCompositionTimeOffset; + } + + public class Trun { + + public int chunkDuration; + public int chunkSize; + + public int bFlags; + int bFirstSampleFlags; + int dataOffset; + + public int entryCount; + byte[] bEntries; + int entries_rowSize; + + public TrunEntry getEntry(int i) { + ByteBuffer buffer = ByteBuffer.wrap(bEntries, i * entries_rowSize, entries_rowSize); + TrunEntry entry = new TrunEntry(); + + if (hasFlag(bFlags, 0x0100)) { + entry.sampleDuration = buffer.getInt(); + } + if (hasFlag(bFlags, 0x0200)) { + entry.sampleSize = buffer.getInt(); + } + if (hasFlag(bFlags, 0x0400)) { + entry.sampleFlags = buffer.getInt(); + } + if (hasFlag(bFlags, 0x0800)) { + entry.sampleCompositionTimeOffset = buffer.getInt(); + } + + return entry; + } + } + + public class Tkhd { + + int trackId; + long duration; + short bVolume; + int bWidth; + int bHeight; + byte[] matrix; + short bLayer; + short bAlternateGroup; + } + + public class Trak { + + public Tkhd tkhd; + public int mdia_mdhd_timeScale; + + byte[] mdia; + } + + class Mvhd { + + long timeScale; + long nextTrackId; + } + + class Moov { + + Mvhd mvhd; + Trak[] trak; + Trex[] mvex_trex; + } + + class Tfra { + + int trackId; + int[] entries_time; + } + + public class Trex { + + private int trackId; + int defaultSampleDescriptionIndex; + int defaultSampleDuration; + int defaultSampleSize; + int defaultSampleFlags; + } + + public class Mp4Track { + + public TrackKind kind; + public Trak trak; + public Trex trex; + } + + public class Mp4TrackChunk { + + public InputStream data; + public Moof moof; + } +// +} diff --git a/app/src/main/java/org/schabi/newpipe/streams/Mp4DashWriter.java b/app/src/main/java/org/schabi/newpipe/streams/Mp4DashWriter.java new file mode 100644 index 000000000..babb2e24c --- /dev/null +++ b/app/src/main/java/org/schabi/newpipe/streams/Mp4DashWriter.java @@ -0,0 +1,623 @@ +package org.schabi.newpipe.streams; + +import org.schabi.newpipe.streams.io.SharpStream; + +import org.schabi.newpipe.streams.Mp4DashReader.Mp4Track; +import org.schabi.newpipe.streams.Mp4DashReader.Mp4TrackChunk; +import org.schabi.newpipe.streams.Mp4DashReader.Trak; +import org.schabi.newpipe.streams.Mp4DashReader.Trex; + + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +import static org.schabi.newpipe.streams.Mp4DashReader.hasFlag; + +/** + * + * @author kapodamy + */ +public class Mp4DashWriter { + + private final static byte DIMENSIONAL_FIVE = 5; + private final static byte DIMENSIONAL_TWO = 2; + private final static short DEFAULT_TIMESCALE = 1000; + private final static int BUFFER_SIZE = 8 * 1024; + private final static byte DEFAULT_TREX_SIZE = 32; + private final static byte[] TFRA_TTS_DEFAULT = new byte[]{0x01, 0x01, 0x01}; + private final static int EPOCH_OFFSET = 2082844800; + + private Mp4Track[] infoTracks; + private SharpStream[] sourceTracks; + + private Mp4DashReader[] readers; + private final long time; + + private boolean done = false; + private boolean parsed = false; + + private long written = 0; + private ArrayList> chunkTimes; + private ArrayList moofOffsets; + private ArrayList fragSizes; + + public Mp4DashWriter(SharpStream... source) { + sourceTracks = source; + readers = new Mp4DashReader[sourceTracks.length]; + infoTracks = new Mp4Track[sourceTracks.length]; + time = (System.currentTimeMillis() / 1000L) + EPOCH_OFFSET; + } + + public Mp4Track[] getTracksFromSource(int sourceIndex) throws IllegalStateException { + if (!parsed) { + throw new IllegalStateException("All sources must be parsed first"); + } + + return readers[sourceIndex].getAvailableTracks(); + } + + public void parseSources() throws IOException, IllegalStateException { + if (done) { + throw new IllegalStateException("already done"); + } + if (parsed) { + throw new IllegalStateException("already parsed"); + } + + try { + for (int i = 0; i < readers.length; i++) { + readers[i] = new Mp4DashReader(sourceTracks[i]); + readers[i].parse(); + } + + } finally { + parsed = true; + } + } + + public void selectTracks(int... trackIndex) throws IOException { + if (done) { + throw new IOException("already done"); + } + if (chunkTimes != null) { + throw new IOException("tracks already selected"); + } + + try { + chunkTimes = new ArrayList<>(readers.length); + moofOffsets = new ArrayList<>(32); + fragSizes = new ArrayList<>(32); + + for (int i = 0; i < readers.length; i++) { + infoTracks[i] = readers[i].selectTrack(trackIndex[i]); + + chunkTimes.add(new ArrayList(32)); + } + + } finally { + parsed = true; + } + } + + public long getBytesWritten() { + return written; + } + + public void build(SharpStream out) throws IOException, RuntimeException { + if (done) { + throw new RuntimeException("already done"); + } + if (!out.canWrite()) { + throw new IOException("the provided output is not writable"); + } + + long sidxOffsets = -1; + int maxFrags = 0; + + for (SharpStream stream : sourceTracks) { + if (!stream.canRewind()) { + sidxOffsets = -2;// sidx not available + } + } + + try { + dump(make_ftyp(), out); + dump(make_moov(), out); + + if (sidxOffsets == -1 && out.canRewind()) { + // + int reserved = 0; + for (Mp4DashReader reader : readers) { + int count = reader.getFragmentsCount(); + if (count > maxFrags) { + maxFrags = count; + } + reserved += 12 + calcSidxBodySize(count); + } + if (maxFrags > 0xFFFF) { + sidxOffsets = -3;// TODO: to many fragments, needs a multi-sidx implementation + } else { + sidxOffsets = written; + dump(make_free(reserved), out); + } + // + } + ArrayList chunks = new ArrayList<>(readers.length); + chunks.add(null); + + int read; + byte[] buffer = new byte[BUFFER_SIZE]; + int sequenceNumber = 1; + + while (true) { + chunks.clear(); + + for (int i = 0; i < readers.length; i++) { + Mp4TrackChunk chunk = readers[i].getNextChunk(); + if (chunk == null || chunk.moof.traf.trun.chunkSize < 1) { + continue; + } + chunk.moof.traf.tfhd.trackId = i + 1; + chunks.add(chunk); + + if (sequenceNumber == 1) { + if (chunk.moof.traf.trun.entryCount > 0 && hasFlag(chunk.moof.traf.trun.bFlags, 0x0800)) { + chunkTimes.get(i).add(chunk.moof.traf.trun.getEntry(0).sampleCompositionTimeOffset); + } else { + chunkTimes.get(i).add(0); + } + } + + chunkTimes.get(i).add(chunk.moof.traf.trun.chunkDuration); + } + + if (chunks.size() < 1) { + break; + } + + long offset = written; + moofOffsets.add(offset); + + dump(make_moof(sequenceNumber++, chunks, offset), out); + dump(make_mdat(chunks), out); + + for (Mp4TrackChunk chunk : chunks) { + while ((read = chunk.data.read(buffer)) > 0) { + out.write(buffer, 0, read); + written += read; + } + } + + fragSizes.add((int) (written - offset)); + } + + dump(make_mfra(), out); + + if (sidxOffsets > 0 && moofOffsets.size() == maxFrags) { + long len = written; + + out.rewind(); + out.skip(sidxOffsets); + + written = sidxOffsets; + sidxOffsets = moofOffsets.get(0); + + for (int i = 0; i < readers.length; i++) { + dump(make_sidx(i, sidxOffsets - written), out); + } + + written = len; + } + } finally { + done = true; + } + } + + public boolean isDone() { + return done; + } + + public boolean isParsed() { + return parsed; + } + + public void close() { + done = true; + parsed = true; + + for (SharpStream src : sourceTracks) { + src.dispose(); + } + + sourceTracks = null; + readers = null; + infoTracks = null; + moofOffsets = null; + chunkTimes = null; + } + + // + private void dump(byte[][] buffer, SharpStream stream) throws IOException { + for (byte[] buff : buffer) { + stream.write(buff); + written += buff.length; + } + } + + private byte[][] lengthFor(byte[][] buffer) { + int length = 0; + for (byte[] buff : buffer) { + length += buff.length; + } + + ByteBuffer.wrap(buffer[0]).putInt(length); + + return buffer; + } + + private int calcSidxBodySize(int entryCount) { + return 4 + 4 + 8 + 8 + 4 + (entryCount * 12); + } + // + + // + private byte[][] make_moof(int sequence, ArrayList chunks, long referenceOffset) { + int pos = 2; + TrunExtra[] extra = new TrunExtra[chunks.size()]; + + byte[][] buffer = new byte[pos + (extra.length * DIMENSIONAL_FIVE)][]; + buffer[0] = new byte[]{ + 0x00, 0x00, 0x00, 0x00, 0x6D, 0x6F, 0x6F, 0x66,// info header + 0x00, 0x00, 0x00, 0x10, 0x6D, 0x66, 0x68, 0x64, 0x00, 0x00, 0x00, 0x00//mfhd + }; + buffer[1] = new byte[4]; + ByteBuffer.wrap(buffer[1]).putInt(sequence); + + for (int i = 0; i < extra.length; i++) { + extra[i] = new TrunExtra(); + for (byte[] buff : make_traf(chunks.get(i), extra[i], referenceOffset)) { + buffer[pos++] = buff; + } + } + + lengthFor(buffer); + + int offset = 8 + ByteBuffer.wrap(buffer[0]).getInt(); + + for (int i = 0; i < extra.length; i++) { + extra[i].byteBuffer.putInt(offset); + offset += chunks.get(i).moof.traf.trun.chunkSize; + } + + return buffer; + } + + private byte[][] make_traf(Mp4TrackChunk chunk, TrunExtra extra, long moofOffset) { + byte[][] buffer = new byte[DIMENSIONAL_FIVE][]; + buffer[0] = new byte[]{ + 0x00, 0x00, 0x00, 0x00, 0x74, 0x72, 0x61, 0x66, + 0x00, 0x00, 0x00, 0x00, 0x74, 0x66, 0x68, 0x64 + }; + + int flags = (chunk.moof.traf.tfhd.bFlags & 0x38) | 0x01; + byte tfhdBodySize = 8 + 8; + if (hasFlag(flags, 0x08)) { + tfhdBodySize += 4; + } + if (hasFlag(flags, 0x10)) { + tfhdBodySize += 4; + } + if (hasFlag(flags, 0x20)) { + tfhdBodySize += 4; + } + buffer[1] = new byte[tfhdBodySize]; + ByteBuffer set = ByteBuffer.wrap(buffer[1]); + set.position(4); + set.putInt(chunk.moof.traf.tfhd.trackId); + set.putLong(moofOffset); + if (hasFlag(flags, 0x08)) { + set.putInt(chunk.moof.traf.tfhd.defaultSampleDuration); + } + if (hasFlag(flags, 0x10)) { + set.putInt(chunk.moof.traf.tfhd.defaultSampleSize); + } + if (hasFlag(flags, 0x20)) { + set.putInt(chunk.moof.traf.tfhd.defaultSampleFlags); + } + set.putInt(0, flags); + ByteBuffer.wrap(buffer[0]).putInt(8, 8 + tfhdBodySize); + + buffer[2] = new byte[]{ + 0x00, 0x00, 0x00, 0x14, + 0x74, 0x66, 0x64, 0x74, + 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00 + }; + + ByteBuffer.wrap(buffer[2]).putLong(12, chunk.moof.traf.tfdt); + + buffer[3] = new byte[]{ + 0x00, 0x00, 0x00, 0x00, 0x74, 0x72, 0x75, 0x6E, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00 + }; + + buffer[4] = chunk.moof.traf.trun.bEntries; + + lengthFor(buffer); + + set = ByteBuffer.wrap(buffer[3]); + set.putInt(buffer[3].length + buffer[4].length); + set.position(8); + set.putInt((chunk.moof.traf.trun.bFlags | 0x01) & 0x0F01); + set.putInt(chunk.moof.traf.trun.entryCount); + extra.byteBuffer = set; + + return buffer; + } + + private byte[][] make_mdat(ArrayList chunks) { + byte[][] buffer = new byte[][]{ + { + 0x00, 0x00, 0x00, 0x00, 0x6D, 0x64, 0x61, 0x74 + } + }; + + int length = 0; + + for (Mp4TrackChunk chunk : chunks) { + length += chunk.moof.traf.trun.chunkSize; + } + + ByteBuffer.wrap(buffer[0]).putInt(length + 8); + + return buffer; + } + + private byte[][] make_ftyp() { + return new byte[][]{ + { + 0x00, 0x00, 0x00, 0x20, 0x66, 0x74, 0x79, 0x70, 0x64, 0x61, 0x73, 0x68, 0x00, 0x00, 0x00, 0x00, + 0x6D, 0x70, 0x34, 0x31, 0x69, 0x73, 0x6F, 0x6D, 0x69, 0x73, 0x6F, 0x36, 0x69, 0x73, 0x6F, 0x32 + } + }; + } + + private byte[][] make_mvhd() { + byte[][] buffer = new byte[DIMENSIONAL_FIVE][]; + + buffer[0] = new byte[]{ + 0x00, 0x00, 0x00, 0x78, 0x6D, 0x76, 0x68, 0x64, 0x01, 0x00, 0x00, 0x00 + }; + buffer[1] = new byte[28]; + buffer[2] = new byte[]{ + 0x00, 0x01, 0x00, 0x00, 0x01, 0x00,// default volume and rate + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,// reserved values + // default matrix + 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x40, 0x00, 0x00, 0x00 + }; + buffer[3] = new byte[24];// predefined + buffer[4] = ByteBuffer.allocate(4).putInt(infoTracks.length + 1).array(); + + long longestTrack = 0; + + for (Mp4Track track : infoTracks) { + long tmp = (long) ((track.trak.tkhd.duration / (double) track.trak.mdia_mdhd_timeScale) * DEFAULT_TIMESCALE); + if (tmp > longestTrack) { + longestTrack = tmp; + } + } + + ByteBuffer.wrap(buffer[1]) + .putLong(time) + .putLong(time) + .putInt(DEFAULT_TIMESCALE) + .putLong(longestTrack); + + return buffer; + } + + private byte[][] make_trak(int trackId, Trak trak) throws RuntimeException { + if (trak.tkhd.matrix.length != 36) { + throw new RuntimeException("bad track matrix length (expected 36)"); + } + + byte[][] buffer = new byte[DIMENSIONAL_FIVE][]; + + buffer[0] = new byte[]{ + 0x00, 0x00, 0x00, 0x00, 0x74, 0x72, 0x61, 0x6B,// trak header + 0x00, 0x00, 0x00, 0x68, 0x74, 0x6B, 0x68, 0x64, 0x01, 0x00, 0x00, 0x03 // tkhd header + }; + buffer[1] = new byte[48]; + buffer[2] = trak.tkhd.matrix; + buffer[3] = new byte[8]; + buffer[4] = trak.mdia; + + ByteBuffer set = ByteBuffer.wrap(buffer[1]); + set.putLong(time); + set.putLong(time); + set.putInt(trackId); + set.position(24); + set.putLong(trak.tkhd.duration); + set.position(40); + set.putShort(trak.tkhd.bLayer); + set.putShort(trak.tkhd.bAlternateGroup); + set.putShort(trak.tkhd.bVolume); + + ByteBuffer.wrap(buffer[3]) + .putInt(trak.tkhd.bWidth) + .putInt(trak.tkhd.bHeight); + + return lengthFor(buffer); + } + + private byte[][] make_moov() throws RuntimeException { + int pos = 1; + byte[][] buffer = new byte[2 + (DIMENSIONAL_TWO * infoTracks.length) + (DIMENSIONAL_FIVE * infoTracks.length) + DIMENSIONAL_FIVE + 1][]; + + buffer[0] = new byte[]{ + 0x00, 0x00, 0x00, 0x00, 0x6D, 0x6F, 0x6F, 0x76 + }; + + for (byte[] buff : make_mvhd()) { + buffer[pos++] = buff; + } + + for (int i = 0; i < infoTracks.length; i++) { + for (byte[] buff : make_trak(i + 1, infoTracks[i].trak)) { + buffer[pos++] = buff; + } + } + + buffer[pos] = new byte[]{ + 0x00, 0x00, 0x00, 0x00, 0x6D, 0x76, 0x65, 0x78 + }; + + ByteBuffer.wrap(buffer[pos++]).putInt((infoTracks.length * DEFAULT_TREX_SIZE) + 8); + + for (int i = 0; i < infoTracks.length; i++) { + for (byte[] buff : make_trex(i + 1, infoTracks[i].trex)) { + buffer[pos++] = buff; + } + } + + // default udta + buffer[pos] = new byte[]{ + 0x00, 0x00, 0x00, 0x5C, 0x75, 0x64, 0x74, 0x61, 0x00, 0x00, 0x00, 0x54, 0x6D, 0x65, 0x74, 0x61, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x21, 0x68, 0x64, 0x6C, 0x72, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x6D, 0x64, 0x69, 0x72, 0x61, 0x70, 0x70, 0x6C, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x27, 0x69, 0x6C, 0x73, 0x74, 0x00, 0x00, 0x00, + 0x1F, (byte) 0xA9, 0x63, 0x6D, 0x74, 0x00, 0x00, 0x00, 0x17, 0x64, 0x61, 0x74, 0x61, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, + 0x4E, 0x65, 0x77, 0x50, 0x69, 0x70, 0x65// "NewPipe" binary string + }; + + return lengthFor(buffer); + } + + private byte[][] make_trex(int trackId, Trex trex) { + byte[][] buffer = new byte[][]{ + { + 0x00, 0x00, 0x00, 0x20, 0x74, 0x72, 0x65, 0x78, 0x00, 0x00, 0x00, 0x00 + }, + new byte[20] + }; + + ByteBuffer.wrap(buffer[1]) + .putInt(trackId) + .putInt(trex.defaultSampleDescriptionIndex) + .putInt(trex.defaultSampleDuration) + .putInt(trex.defaultSampleSize) + .putInt(trex.defaultSampleFlags); + + return buffer; + } + + private byte[][] make_tfra(int trackId, List times, List moofOffsets) { + int entryCount = times.size() - 1; + byte[][] buffer = new byte[DIMENSIONAL_TWO][]; + buffer[0] = new byte[]{ + 0x00, 0x00, 0x00, 0x00, 0x74, 0x66, 0x72, 0x61, 0x01, 0x00, 0x00, 0x00 + }; + buffer[1] = new byte[12 + ((16 + TFRA_TTS_DEFAULT.length) * entryCount)]; + + ByteBuffer set = ByteBuffer.wrap(buffer[1]); + set.putInt(trackId); + set.position(8); + set.putInt(entryCount); + + long decodeTime = 0; + + for (int i = 0; i < entryCount; i++) { + decodeTime += times.get(i); + set.putLong(decodeTime); + set.putLong(moofOffsets.get(i)); + set.put(TFRA_TTS_DEFAULT);// default values: traf number/trun number/sample number + } + + return lengthFor(buffer); + } + + private byte[][] make_mfra() { + byte[][] buffer = new byte[2 + (DIMENSIONAL_TWO * infoTracks.length)][]; + buffer[0] = new byte[]{ + 0x00, 0x00, 0x00, 0x00, 0x6D, 0x66, 0x72, 0x61 + }; + int pos = 1; + + for (int i = 0; i < infoTracks.length; i++) { + for (byte[] buff : make_tfra(i + 1, chunkTimes.get(i), moofOffsets)) { + buffer[pos++] = buff; + } + } + + buffer[pos] = new byte[]{// mfro + 0x00, 0x00, 0x00, 0x10, 0x6D, 0x66, 0x72, 0x6F, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 + }; + + lengthFor(buffer); + + ByteBuffer set = ByteBuffer.wrap(buffer[pos]); + set.position(12); + set.put(buffer[0], 0, 4); + + return buffer; + + } + + private byte[][] make_sidx(int internalTrackId, long firstOffset) { + List times = chunkTimes.get(internalTrackId); + int count = times.size() - 1;// the first item is ignored (composition time) + + if (count > 65535) { + throw new OutOfMemoryError("to many fragments. sidx limit is 65535, found " + String.valueOf(count)); + } + + byte[][] buffer = new byte[][]{ + new byte[]{ + 0x00, 0x00, 0x00, 0x00, 0x73, 0x69, 0x64, 0x78, 0x01, 0x00, 0x00, 0x00 + }, + new byte[calcSidxBodySize(count)] + }; + + lengthFor(buffer); + + ByteBuffer set = ByteBuffer.wrap(buffer[1]); + set.putInt(internalTrackId + 1); + set.putInt(infoTracks[internalTrackId].trak.mdia_mdhd_timeScale); + set.putLong(0); + set.putLong(firstOffset - ByteBuffer.wrap(buffer[0]).getInt()); + set.putInt(0xFFFF & count);// unsigned + + int i = 0; + while (i < count) { + set.putInt(fragSizes.get(i) & 0x7fffffff);// default reference type is 0 + set.putInt(times.get(i + 1)); + set.putInt(0x90000000);// default SAP settings + i++; + } + + return buffer; + } + + private byte[][] make_free(int totalSize) { + return lengthFor(new byte[][]{ + new byte[]{0x00, 0x00, 0x00, 0x00, 0x66, 0x72, 0x65, 0x65}, + new byte[totalSize - 8]// this is waste of RAM + }); + + } + +// + + class TrunExtra { + + ByteBuffer byteBuffer; + } +} diff --git a/app/src/main/java/org/schabi/newpipe/streams/SubtitleConverter.java b/app/src/main/java/org/schabi/newpipe/streams/SubtitleConverter.java new file mode 100644 index 000000000..26aaf49a5 --- /dev/null +++ b/app/src/main/java/org/schabi/newpipe/streams/SubtitleConverter.java @@ -0,0 +1,370 @@ +package org.schabi.newpipe.streams; + +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; +import org.xml.sax.SAXException; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.nio.charset.Charset; +import java.text.ParseException; +import java.util.Locale; + +import org.schabi.newpipe.streams.io.SharpStream; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.xpath.XPathExpressionException; + +/** + * @author kapodamy + */ +public class SubtitleConverter { + private static final String NEW_LINE = "\r\n"; + + public void dumpTTML(SharpStream in, final SharpStream out, final boolean ignoreEmptyFrames, final boolean detectYoutubeDuplicateLines + ) throws IOException, ParseException, SAXException, ParserConfigurationException, XPathExpressionException { + + final FrameWriter callback = new FrameWriter() { + int frameIndex = 0; + final Charset charset = Charset.forName("utf-8"); + + @Override + public void yield(SubtitleFrame frame) throws IOException { + if (ignoreEmptyFrames && frame.isEmptyText()) { + return; + } + out.write(String.valueOf(frameIndex++).getBytes(charset)); + out.write(NEW_LINE.getBytes(charset)); + out.write(getTime(frame.start, true).getBytes(charset)); + out.write(" --> ".getBytes(charset)); + out.write(getTime(frame.end, true).getBytes(charset)); + out.write(NEW_LINE.getBytes(charset)); + out.write(frame.text.getBytes(charset)); + out.write(NEW_LINE.getBytes(charset)); + out.write(NEW_LINE.getBytes(charset)); + } + }; + + read_xml_based(in, callback, detectYoutubeDuplicateLines, + "tt", "xmlns", "http://www.w3.org/ns/ttml", + new String[]{"timedtext", "head", "wp"}, + new String[]{"body", "div", "p"}, + "begin", "end", true + ); + } + + private void read_xml_based(SharpStream source, FrameWriter callback, boolean detectYoutubeDuplicateLines, + String root, String formatAttr, String formatVersion, String[] cuePath, String[] framePath, + String timeAttr, String durationAttr, boolean hasTimestamp + ) throws IOException, ParseException, SAXException, ParserConfigurationException, XPathExpressionException { + /* + * XML based subtitles parser with BASIC support + * multiple CUE is not supported + * styling is not supported + * tag timestamps (in auto-generated subtitles) are not supported, maybe in the future + * also TimestampTagOption enum is not applicable + * Language parsing is not supported + */ + + byte[] buffer = new byte[source.available()]; + source.read(buffer); + + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + factory.setNamespaceAware(true); + DocumentBuilder builder = factory.newDocumentBuilder(); + Document xml = builder.parse(new ByteArrayInputStream(buffer)); + + String attr; + + // get the format version or namespace + Element node = xml.getDocumentElement(); + + if (node == null) { + throw new ParseException("Can't get the format version. ¿wrong namespace?", -1); + } else if (!node.getNodeName().equals(root)) { + throw new ParseException("Invalid root", -1); + } + + if (formatAttr.equals("xmlns")) { + if (!node.getNamespaceURI().equals(formatVersion)) { + throw new UnsupportedOperationException("Expected xml namespace: " + formatVersion); + } + } else { + attr = node.getAttributeNS(formatVersion, formatAttr); + if (attr == null) { + throw new ParseException("Can't get the format attribute", -1); + } + if (!attr.equals(formatVersion)) { + throw new ParseException("Invalid format version : " + attr, -1); + } + } + + NodeList node_list; + + int line_break = 0;// Maximum characters per line if present (valid for TranScript v3) + + if (!hasTimestamp) { + node_list = selectNodes(xml, cuePath, formatVersion); + + if (node_list != null) { + // if the subtitle has multiple CUEs, use the highest value + for (int i = 0; i < node_list.getLength(); i++) { + try { + int tmp = Integer.parseInt(((Element) node_list.item(i)).getAttributeNS(formatVersion, "ah")); + if (tmp > line_break) { + line_break = tmp; + } + } catch (Exception err) { + } + } + } + } + + // parse every frame + node_list = selectNodes(xml, framePath, formatVersion); + + if (node_list == null) { + return;// no frames detected + } + + int fs_ff = -1;// first timestamp of first frame + boolean limit_lines = false; + + for (int i = 0; i < node_list.getLength(); i++) { + Element elem = (Element) node_list.item(i); + SubtitleFrame obj = new SubtitleFrame(); + obj.text = elem.getTextContent(); + + attr = elem.getAttribute(timeAttr);// ¡this cant be null! + obj.start = hasTimestamp ? parseTimestamp(attr) : Integer.parseInt(attr); + + attr = elem.getAttribute(durationAttr); + if (obj.text == null || attr == null) { + continue;// normally is a blank line (on auto-generated subtitles) ignore + } + + if (hasTimestamp) { + obj.end = parseTimestamp(attr); + + if (detectYoutubeDuplicateLines) { + if (limit_lines) { + int swap = obj.end; + obj.end = fs_ff; + fs_ff = swap; + } else { + if (fs_ff < 0) { + fs_ff = obj.end; + } else { + if (fs_ff < obj.start) { + limit_lines = true;// the subtitles has duplicated lines + } else { + detectYoutubeDuplicateLines = false; + } + } + } + } + } else { + obj.end = obj.start + Integer.parseInt(attr); + } + + if (/*node.getAttribute("w").equals("1") &&*/line_break > 1 && obj.text.length() > line_break) { + + // implement auto line breaking (once) + StringBuilder text = new StringBuilder(obj.text); + obj.text = null; + + switch (text.charAt(line_break)) { + case ' ': + case '\t': + putBreakAt(line_break, text); + break; + default:// find the word start position + for (int j = line_break - 1; j > 0; j--) { + switch (text.charAt(j)) { + case ' ': + case '\t': + putBreakAt(j, text); + j = -1; + break; + case '\r': + case '\n': + j = -1;// long word, just ignore + break; + } + } + break; + } + + obj.text = text.toString();// set the processed text + } + + callback.yield(obj); + } + } + + private static NodeList selectNodes(Document xml, String[] path, String namespaceUri) throws XPathExpressionException { + Element ref = xml.getDocumentElement(); + + for (int i = 0; i < path.length - 1; i++) { + NodeList nodes = ref.getChildNodes(); + if (nodes.getLength() < 1) { + return null; + } + + Element elem; + for (int j = 0; j < nodes.getLength(); j++) { + if (nodes.item(j).getNodeType() == Node.ELEMENT_NODE) { + elem = (Element) nodes.item(j); + if (elem.getNodeName().equals(path[i]) && elem.getNamespaceURI().equals(namespaceUri)) { + ref = elem; + break; + } + } + } + } + + return ref.getElementsByTagNameNS(namespaceUri, path[path.length - 1]); + } + + private static int parseTimestamp(String multiImpl) throws NumberFormatException, ParseException { + if (multiImpl.length() < 1) { + return 0; + } else if (multiImpl.length() == 1) { + return Integer.parseInt(multiImpl) * 1000;// ¡this must be a number in seconds! + } + + // detect wallclock-time + if (multiImpl.startsWith("wallclock(")) { + throw new UnsupportedOperationException("Parsing wallclock timestamp is not implemented"); + } + + // detect offset-time + if (multiImpl.indexOf(':') < 0) { + int multiplier = 1000; + char metric = multiImpl.charAt(multiImpl.length() - 1); + switch (metric) { + case 'h': + multiplier *= 3600000; + break; + case 'm': + multiplier *= 60000; + break; + case 's': + if (multiImpl.charAt(multiImpl.length() - 2) == 'm') { + multiplier = 1;// ms + } + break; + default: + if (!Character.isDigit(metric)) { + throw new NumberFormatException("Invalid metric suffix found on : " + multiImpl); + } + metric = '\0'; + break; + } + try { + String offset_time = multiImpl; + + if (multiplier == 1) { + offset_time = offset_time.substring(0, offset_time.length() - 2); + } else if (metric != '\0') { + offset_time = offset_time.substring(0, offset_time.length() - 1); + } + + double time_metric_based = Double.parseDouble(offset_time); + if (Math.abs(time_metric_based) <= Double.MAX_VALUE) { + return (int) (time_metric_based * multiplier); + } + } catch (Exception err) { + throw new UnsupportedOperationException("Invalid or not implemented timestamp on: " + multiImpl); + } + } + + // detect clock-time + int time = 0; + String[] units = multiImpl.split(":"); + + if (units.length < 3) { + throw new ParseException("Invalid clock-time timestamp", -1); + } + + time += Integer.parseInt(units[0]) * 3600000;// hours + time += Integer.parseInt(units[1]) * 60000;//minutes + time += Float.parseFloat(units[2]) * 1000f;// seconds and milliseconds (if present) + + // frames and sub-frames are ignored (not implemented) + // time += units[3] * fps; + return time; + } + + private static void putBreakAt(int idx, StringBuilder str) { + // this should be optimized at compile time + + if (NEW_LINE.length() > 1) { + str.delete(idx, idx + 1);// remove after replace + str.insert(idx, NEW_LINE); + } else { + str.setCharAt(idx, NEW_LINE.charAt(0)); + } + } + + private static String getTime(int time, boolean comma) { + // cast every value to integer to avoid auto-round in ToString("00"). + StringBuilder str = new StringBuilder(12); + str.append(numberToString(time / 1000 / 3600, 2));// hours + str.append(':'); + str.append(numberToString(time / 1000 / 60 % 60, 2));// minutes + str.append(':'); + str.append(numberToString(time / 1000 % 60, 2));// seconds + str.append(comma ? ',' : '.'); + str.append(numberToString(time % 1000, 3));// miliseconds + + return str.toString(); + } + + private static String numberToString(int nro, int pad) { + return String.format(Locale.ENGLISH, "%0".concat(String.valueOf(pad)).concat("d"), nro); + } + + + /****************** + * helper classes * + ******************/ + + private interface FrameWriter { + + void yield(SubtitleFrame frame) throws IOException; + } + + private static class SubtitleFrame { + //Java no support unsigned int + + public int end; + public int start; + public String text = ""; + + private boolean isEmptyText() { + if (text == null) { + return true; + } + + for (int i = 0; i < text.length(); i++) { + switch (text.charAt(i)) { + case ' ': + case '\t': + case '\r': + case '\n': + break; + default: + return false; + } + } + + return true; + } + } + +} diff --git a/app/src/main/java/org/schabi/newpipe/streams/TrackDataChunk.java b/app/src/main/java/org/schabi/newpipe/streams/TrackDataChunk.java new file mode 100644 index 000000000..86eb5ff4f --- /dev/null +++ b/app/src/main/java/org/schabi/newpipe/streams/TrackDataChunk.java @@ -0,0 +1,65 @@ +package org.schabi.newpipe.streams; + +import java.io.InputStream; +import java.io.IOException; + +public class TrackDataChunk extends InputStream { + + private final DataReader base; + private int size; + + public TrackDataChunk(DataReader base, int size) { + this.base = base; + this.size = size; + } + + @Override + public int read() throws IOException { + if (size < 1) { + return -1; + } + + int res = base.read(); + + if (res >= 0) { + size--; + } + + return res; + } + + @Override + public int read(byte[] buffer) throws IOException { + return read(buffer, 0, buffer.length); + } + + @Override + public int read(byte[] buffer, int offset, int count) throws IOException { + count = Math.min(size, count); + int read = base.read(buffer, offset, count); + size -= count; + return read; + } + + @Override + public long skip(long amount) throws IOException { + long res = base.skipBytes(Math.min(amount, size)); + size -= res; + return res; + } + + @Override + public int available() { + return size; + } + + @Override + public void close() { + size = 0; + } + + @Override + public boolean markSupported() { + return false; + } +} diff --git a/app/src/main/java/org/schabi/newpipe/streams/WebMReader.java b/app/src/main/java/org/schabi/newpipe/streams/WebMReader.java new file mode 100644 index 000000000..f61ef14c5 --- /dev/null +++ b/app/src/main/java/org/schabi/newpipe/streams/WebMReader.java @@ -0,0 +1,507 @@ +package org.schabi.newpipe.streams; + +import java.io.EOFException; +import java.io.IOException; +import java.util.ArrayList; +import java.util.NoSuchElementException; +import java.util.Objects; + +import org.schabi.newpipe.streams.io.SharpStream; + +/** + * + * @author kapodamy + */ +public class WebMReader { + + // + private final static int ID_EMBL = 0x0A45DFA3; + private final static int ID_EMBLReadVersion = 0x02F7; + private final static int ID_EMBLDocType = 0x0282; + private final static int ID_EMBLDocTypeReadVersion = 0x0285; + + private final static int ID_Segment = 0x08538067; + + private final static int ID_Info = 0x0549A966; + private final static int ID_TimecodeScale = 0x0AD7B1; + private final static int ID_Duration = 0x489; + + private final static int ID_Tracks = 0x0654AE6B; + private final static int ID_TrackEntry = 0x2E; + private final static int ID_TrackNumber = 0x57; + private final static int ID_TrackType = 0x03; + private final static int ID_CodecID = 0x06; + private final static int ID_CodecPrivate = 0x23A2; + private final static int ID_Video = 0x60; + private final static int ID_Audio = 0x61; + private final static int ID_DefaultDuration = 0x3E383; + private final static int ID_FlagLacing = 0x1C; + + private final static int ID_Cluster = 0x0F43B675; + private final static int ID_Timecode = 0x67; + private final static int ID_SimpleBlock = 0x23; +// + + public enum TrackKind { + Audio/*2*/, Video/*1*/, Other + } + + private DataReader stream; + private Segment segment; + private WebMTrack[] tracks; + private int selectedTrack; + private boolean done; + private boolean firstSegment; + + public WebMReader(SharpStream source) { + this.stream = new DataReader(source); + } + + public void parse() throws IOException { + Element elem = readElement(ID_EMBL); + if (!readEbml(elem, 1, 2)) { + throw new UnsupportedOperationException("Unsupported EBML data (WebM)"); + } + ensure(elem); + + elem = untilElement(null, ID_Segment); + if (elem == null) { + throw new IOException("Fragment element not found"); + } + segment = readSegment(elem, 0, true); + tracks = segment.tracks; + selectedTrack = -1; + done = false; + firstSegment = true; + } + + public WebMTrack[] getAvailableTracks() { + return tracks; + } + + public WebMTrack selectTrack(int index) { + selectedTrack = index; + return tracks[index]; + } + + public Segment getNextSegment() throws IOException { + if (done) { + return null; + } + + if (firstSegment && segment != null) { + firstSegment = false; + return segment; + } + + ensure(segment.ref); + + Element elem = untilElement(null, ID_Segment); + if (elem == null) { + done = true; + return null; + } + segment = readSegment(elem, 0, false); + + return segment; + } + + // + private long readNumber(Element parent) throws IOException { + int length = (int) parent.contentSize; + long value = 0; + while (length-- > 0) { + int read = stream.read(); + if (read == -1) { + throw new EOFException(); + } + value = (value << 8) | read; + } + return value; + } + + private String readString(Element parent) throws IOException { + return new String(readBlob(parent), "utf-8"); + } + + private byte[] readBlob(Element parent) throws IOException { + long length = parent.contentSize; + byte[] buffer = new byte[(int) length]; + int read = stream.read(buffer); + if (read < length) { + throw new EOFException(); + } + return buffer; + } + + private long readEncodedNumber() throws IOException { + int value = stream.read(); + + if (value > 0) { + byte size = 1; + int mask = 0x80; + + while (size < 9) { + if ((value & mask) == mask) { + mask = 0xFF; + mask >>= size; + + long number = value & mask; + + for (int i = 1; i < size; i++) { + value = stream.read(); + number <<= 8; + number |= value; + } + + return number; + } + + mask >>= 1; + size++; + } + } + + throw new IOException("Invalid encoded length"); + } + + private Element readElement() throws IOException { + Element elem = new Element(); + elem.offset = stream.position(); + elem.type = (int) readEncodedNumber(); + elem.contentSize = readEncodedNumber(); + elem.size = elem.contentSize + stream.position() - elem.offset; + + return elem; + } + + private Element readElement(int expected) throws IOException { + Element elem = readElement(); + if (expected != 0 && elem.type != expected) { + throw new NoSuchElementException("expected " + elementID(expected) + " found " + elementID(elem.type)); + } + + return elem; + } + + private Element untilElement(Element ref, int... expected) throws IOException { + Element elem; + while (ref == null ? stream.available() : (stream.position() < (ref.offset + ref.size))) { + elem = readElement(); + for (int type : expected) { + if (elem.type == type) { + return elem; + } + } + ensure(elem); + } + + return null; + } + + private String elementID(long type) { + return "0x".concat(Long.toHexString(type)); + } + + private void ensure(Element ref) throws IOException { + long skip = (ref.offset + ref.size) - stream.position(); + + if (skip == 0) { + return; + } else if (skip < 0) { + throw new EOFException(String.format( + "parser go beyond limits of the Element. type=%s offset=%s size=%s position=%s", + elementID(ref.type), ref.offset, ref.size, stream.position() + )); + } + + stream.skipBytes(skip); + } +// + + // + private boolean readEbml(Element ref, int minReadVersion, int minDocTypeVersion) throws IOException { + Element elem = untilElement(ref, ID_EMBLReadVersion); + if (elem == null) { + return false; + } + if (readNumber(elem) > minReadVersion) { + return false; + } + + elem = untilElement(ref, ID_EMBLDocType); + if (elem == null) { + return false; + } + if (!readString(elem).equals("webm")) { + return false; + } + elem = untilElement(ref, ID_EMBLDocTypeReadVersion); + + return elem != null && readNumber(elem) <= minDocTypeVersion; + } + + private Info readInfo(Element ref) throws IOException { + Element elem; + Info info = new Info(); + + while ((elem = untilElement(ref, ID_TimecodeScale, ID_Duration)) != null) { + switch (elem.type) { + case ID_TimecodeScale: + info.timecodeScale = readNumber(elem); + break; + case ID_Duration: + info.duration = readNumber(elem); + break; + } + ensure(elem); + } + + if (info.timecodeScale == 0) { + throw new NoSuchElementException("Element Timecode not found"); + } + + return info; + } + + private Segment readSegment(Element ref, int trackLacingExpected, boolean metadataExpected) throws IOException { + Segment obj = new Segment(ref); + Element elem; + while ((elem = untilElement(ref, ID_Info, ID_Tracks, ID_Cluster)) != null) { + if (elem.type == ID_Cluster) { + obj.currentCluster = elem; + break; + } + switch (elem.type) { + case ID_Info: + obj.info = readInfo(elem); + break; + case ID_Tracks: + obj.tracks = readTracks(elem, trackLacingExpected); + break; + } + ensure(elem); + } + + if (metadataExpected && (obj.info == null || obj.tracks == null)) { + throw new RuntimeException("Cluster element found without Info and/or Tracks element at position " + String.valueOf(ref.offset)); + } + + return obj; + } + + private WebMTrack[] readTracks(Element ref, int lacingExpected) throws IOException { + ArrayList trackEntries = new ArrayList<>(2); + Element elem_trackEntry; + + while ((elem_trackEntry = untilElement(ref, ID_TrackEntry)) != null) { + WebMTrack entry = new WebMTrack(); + boolean drop = false; + Element elem; + while ((elem = untilElement(elem_trackEntry, + ID_TrackNumber, ID_TrackType, ID_CodecID, ID_CodecPrivate, ID_FlagLacing, ID_DefaultDuration, ID_Audio, ID_Video + )) != null) { + switch (elem.type) { + case ID_TrackNumber: + entry.trackNumber = readNumber(elem); + break; + case ID_TrackType: + entry.trackType = (int)readNumber(elem); + break; + case ID_CodecID: + entry.codecId = readString(elem); + break; + case ID_CodecPrivate: + entry.codecPrivate = readBlob(elem); + break; + case ID_Audio: + case ID_Video: + entry.bMetadata = readBlob(elem); + break; + case ID_DefaultDuration: + entry.defaultDuration = readNumber(elem); + break; + case ID_FlagLacing: + drop = readNumber(elem) != lacingExpected; + break; + default: + System.out.println(); + break; + } + ensure(elem); + } + if (!drop) { + trackEntries.add(entry); + } + ensure(elem_trackEntry); + } + + WebMTrack[] entries = new WebMTrack[trackEntries.size()]; + trackEntries.toArray(entries); + + for (WebMTrack entry : entries) { + switch (entry.trackType) { + case 1: + entry.kind = TrackKind.Video; + break; + case 2: + entry.kind = TrackKind.Audio; + break; + default: + entry.kind = TrackKind.Other; + break; + } + } + + return entries; + } + + private SimpleBlock readSimpleBlock(Element ref) throws IOException { + SimpleBlock obj = new SimpleBlock(ref); + obj.dataSize = stream.position(); + obj.trackNumber = readEncodedNumber(); + obj.relativeTimeCode = stream.readShort(); + obj.flags = (byte) stream.read(); + obj.dataSize = (ref.offset + ref.size) - stream.position(); + + if (obj.dataSize < 0) { + throw new IOException(String.format("Unexpected SimpleBlock element size, missing %s bytes", -obj.dataSize)); + } + return obj; + } + + private Cluster readCluster(Element ref) throws IOException { + Cluster obj = new Cluster(ref); + + Element elem = untilElement(ref, ID_Timecode); + if (elem == null) { + throw new NoSuchElementException("Cluster at " + String.valueOf(ref.offset) + " without Timecode element"); + } + obj.timecode = readNumber(elem); + + return obj; + } +// + + // + class Element { + + int type; + long offset; + long contentSize; + long size; + } + + public class Info { + + public long timecodeScale; + public long duration; + } + + public class WebMTrack { + + public long trackNumber; + protected int trackType; + public String codecId; + public byte[] codecPrivate; + public byte[] bMetadata; + public TrackKind kind; + public long defaultDuration; + } + + public class Segment { + + Segment(Element ref) { + this.ref = ref; + this.firstClusterInSegment = true; + } + + public Info info; + WebMTrack[] tracks; + private Element currentCluster; + private final Element ref; + boolean firstClusterInSegment; + + public Cluster getNextCluster() throws IOException { + if (done) { + return null; + } + if (firstClusterInSegment && segment.currentCluster != null) { + firstClusterInSegment = false; + return readCluster(segment.currentCluster); + } + ensure(segment.currentCluster); + + Element elem = untilElement(segment.ref, ID_Cluster); + if (elem == null) { + return null; + } + + segment.currentCluster = elem; + + return readCluster(segment.currentCluster); + } + } + + public class SimpleBlock { + + public TrackDataChunk data; + + SimpleBlock(Element ref) { + this.ref = ref; + } + + public long trackNumber; + public short relativeTimeCode; + public byte flags; + public long dataSize; + private final Element ref; + + public boolean isKeyframe() { + return (flags & 0x80) == 0x80; + } + } + + public class Cluster { + + Element ref; + SimpleBlock currentSimpleBlock = null; + public long timecode; + + Cluster(Element ref) { + this.ref = ref; + } + + boolean check() { + return stream.position() >= (ref.offset + ref.size); + } + + public SimpleBlock getNextSimpleBlock() throws IOException { + if (check()) { + return null; + } + if (currentSimpleBlock != null) { + ensure(currentSimpleBlock.ref); + } + + while (!check()) { + Element elem = untilElement(ref, ID_SimpleBlock); + if (elem == null) { + return null; + } + + currentSimpleBlock = readSimpleBlock(elem); + if (currentSimpleBlock.trackNumber == tracks[selectedTrack].trackNumber) { + currentSimpleBlock.data = new TrackDataChunk(stream, (int) currentSimpleBlock.dataSize); + return currentSimpleBlock; + } + + ensure(elem); + } + + return null; + } + + } +// +} diff --git a/app/src/main/java/org/schabi/newpipe/streams/WebMWriter.java b/app/src/main/java/org/schabi/newpipe/streams/WebMWriter.java new file mode 100644 index 000000000..ea038c607 --- /dev/null +++ b/app/src/main/java/org/schabi/newpipe/streams/WebMWriter.java @@ -0,0 +1,728 @@ +package org.schabi.newpipe.streams; + +import org.schabi.newpipe.streams.WebMReader.Cluster; +import org.schabi.newpipe.streams.WebMReader.Segment; +import org.schabi.newpipe.streams.WebMReader.SimpleBlock; +import org.schabi.newpipe.streams.WebMReader.WebMTrack; + +import java.io.IOException; +import java.io.InputStream; +import java.io.UnsupportedEncodingException; +import java.nio.ByteBuffer; +import java.util.ArrayList; + +import org.schabi.newpipe.streams.io.SharpStream; + +/** + * + * @author kapodamy + */ +public class WebMWriter { + + private final static int BUFFER_SIZE = 8 * 1024; + private final static int DEFAULT_TIMECODE_SCALE = 1000000; + private final static int INTERV = 100;// 100ms on 1000000us timecode scale + private final static int DEFAULT_CUES_EACH_MS = 5000;// 100ms on 1000000us timecode scale + + private WebMReader.WebMTrack[] infoTracks; + private SharpStream[] sourceTracks; + + private WebMReader[] readers; + + private boolean done = false; + private boolean parsed = false; + + private long written = 0; + + private Segment[] readersSegment; + private Cluster[] readersCluter; + + private int[] predefinedDurations; + + private byte[] outBuffer; + + public WebMWriter(SharpStream... source) { + sourceTracks = source; + readers = new WebMReader[sourceTracks.length]; + infoTracks = new WebMTrack[sourceTracks.length]; + outBuffer = new byte[BUFFER_SIZE]; + } + + public WebMTrack[] getTracksFromSource(int sourceIndex) throws IllegalStateException { + if (done) { + throw new IllegalStateException("already done"); + } + if (!parsed) { + throw new IllegalStateException("All sources must be parsed first"); + } + + return readers[sourceIndex].getAvailableTracks(); + } + + public void parseSources() throws IOException, IllegalStateException { + if (done) { + throw new IllegalStateException("already done"); + } + if (parsed) { + throw new IllegalStateException("already parsed"); + } + + try { + for (int i = 0; i < readers.length; i++) { + readers[i] = new WebMReader(sourceTracks[i]); + readers[i].parse(); + } + + } finally { + parsed = true; + } + } + + public void selectTracks(int... trackIndex) throws IOException { + try { + readersSegment = new Segment[readers.length]; + readersCluter = new Cluster[readers.length]; + predefinedDurations = new int[readers.length]; + + for (int i = 0; i < readers.length; i++) { + infoTracks[i] = readers[i].selectTrack(trackIndex[i]); + predefinedDurations[i] = -1; + readersSegment[i] = readers[i].getNextSegment(); + } + } finally { + parsed = true; + } + } + + public long getBytesWritten() { + return written; + } + + public boolean isDone() { + return done; + } + + public boolean isParsed() { + return parsed; + } + + public void close() { + done = true; + parsed = true; + + for (SharpStream src : sourceTracks) { + src.dispose(); + } + + sourceTracks = null; + readers = null; + infoTracks = null; + readersSegment = null; + readersCluter = null; + outBuffer = null; + } + + public void build(SharpStream out) throws IOException, RuntimeException { + if (!out.canRewind()) { + throw new IOException("The output stream must be allow seek"); + } + + makeEBML(out); + + long offsetSegmentSizeSet = written + 5; + long offsetInfoDurationSet = written + 94; + long offsetClusterSet = written + 58; + long offsetCuesSet = written + 75; + + ArrayList listBuffer = new ArrayList<>(4); + + /* segment */ + listBuffer.add(new byte[]{ + 0x18, 0x53, (byte) 0x80, 0x67, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00// segment content size + }); + + long baseSegmentOffset = written + listBuffer.get(0).length; + + /* seek head */ + listBuffer.add(new byte[]{ + 0x11, 0x4d, (byte) 0x9b, 0x74, (byte) 0xbe, + 0x4d, (byte) 0xbb, (byte) 0x8b, + 0x53, (byte) 0xab, (byte) 0x84, 0x15, 0x49, (byte) 0xa9, 0x66, 0x53, + (byte) 0xac, (byte) 0x81, /*info offset*/ 0x43, + 0x4d, (byte) 0xbb, (byte) 0x8b, 0x53, (byte) 0xab, + (byte) 0x84, 0x16, 0x54, (byte) 0xae, 0x6b, 0x53, (byte) 0xac, (byte) 0x81, + /*tracks offset*/ 0x6a, + 0x4d, (byte) 0xbb, (byte) 0x8e, 0x53, (byte) 0xab, (byte) 0x84, 0x1f, + 0x43, (byte) 0xb6, 0x75, 0x53, (byte) 0xac, (byte) 0x84, /*cluster offset [2]*/ 0x00, 0x00, 0x00, 0x00, + 0x4d, (byte) 0xbb, (byte) 0x8e, 0x53, (byte) 0xab, (byte) 0x84, 0x1c, 0x53, + (byte) 0xbb, 0x6b, 0x53, (byte) 0xac, (byte) 0x84, /*cues offset [7]*/ 0x00, 0x00, 0x00, 0x00 + }); + + /* info */ + listBuffer.add(new byte[]{ + 0x15, 0x49, (byte) 0xa9, 0x66, (byte) 0xa2, 0x2a, (byte) 0xd7, (byte) 0xb1 + }); + listBuffer.add(encode(DEFAULT_TIMECODE_SCALE, true));// this value MUST NOT exceed 4 bytes + listBuffer.add(new byte[]{0x44, (byte) 0x89, (byte) 0x84, + 0x00, 0x00, 0x00, 0x00,// info.duration + + /* MuxingApp */ + 0x4d, (byte) 0x80, (byte) 0x87, 0x4E, + 0x65, 0x77, 0x50, 0x69, 0x70, 0x65, // "NewPipe" binary string + + /* WritingApp */ + 0x57, 0x41, (byte) 0x87, 0x4E, + 0x65, 0x77, 0x50, 0x69, 0x70, 0x65// "NewPipe" binary string + }); + + /* tracks */ + listBuffer.addAll(makeTracks()); + + for (byte[] buff : listBuffer) { + dump(buff, out); + } + + // reserve space for Cues element, but is a waste of space (actually is 64 KiB) + // TODO: better Cue maker + long cueReservedOffset = written; + dump(new byte[]{(byte) 0xec, 0x20, (byte) 0xff, (byte) 0xfb}, out); + int reserved = (1024 * 63) - 4; + while (reserved > 0) { + int write = Math.min(reserved, outBuffer.length); + out.write(outBuffer, 0, write); + reserved -= write; + written += write; + } + + // Select a track for the cue + int cuesForTrackId = selectTrackForCue(); + long nextCueTime = infoTracks[cuesForTrackId].trackType == 1 ? -1 : 0; + ArrayList keyFrames = new ArrayList<>(32); + + //ArrayList chunks = new ArrayList<>(readers.length); + ArrayList clusterOffsets = new ArrayList<>(32); + ArrayList clusterSizes = new ArrayList<>(32); + + long duration = 0; + int durationFromTrackId = 0; + + byte[] bTimecode = makeTimecode(0); + + int firstClusterOffset = (int) written; + long currentClusterOffset = makeCluster(out, bTimecode, 0, clusterOffsets, clusterSizes); + + long baseTimecode = 0; + long limitTimecode = -1; + int limitTimecodeByTrackId = cuesForTrackId; + + int blockWritten = Integer.MAX_VALUE; + + int newClusterByTrackId = -1; + + while (blockWritten > 0) { + blockWritten = 0; + int i = 0; + while (i < readers.length) { + Block bloq = getNextBlockFrom(i); + if (bloq == null) { + i++; + continue; + } + + if (bloq.data == null) { + blockWritten = 1;// fake block + newClusterByTrackId = i; + i++; + continue; + } + + if (newClusterByTrackId == i) { + limitTimecodeByTrackId = i; + newClusterByTrackId = -1; + baseTimecode = bloq.absoluteTimecode; + limitTimecode = baseTimecode + INTERV; + bTimecode = makeTimecode(baseTimecode); + currentClusterOffset = makeCluster(out, bTimecode, currentClusterOffset, clusterOffsets, clusterSizes); + } + + if (cuesForTrackId == i) { + if ((nextCueTime > -1 && bloq.absoluteTimecode >= nextCueTime) || (nextCueTime < 0 && bloq.isKeyframe())) { + if (nextCueTime > -1) { + nextCueTime += DEFAULT_CUES_EACH_MS; + } + keyFrames.add( + new KeyFrame(baseSegmentOffset, currentClusterOffset - 7, written, bTimecode.length, bloq.absoluteTimecode) + ); + } + } + + writeBlock(out, bloq, baseTimecode); + blockWritten++; + + if (bloq.absoluteTimecode > duration) { + duration = bloq.absoluteTimecode; + durationFromTrackId = bloq.trackNumber; + } + + if (limitTimecode < 0) { + limitTimecode = bloq.absoluteTimecode + INTERV; + continue; + } + + if (bloq.absoluteTimecode >= limitTimecode) { + if (limitTimecodeByTrackId != i) { + limitTimecode += INTERV - (bloq.absoluteTimecode - limitTimecode); + } + i++; + } + } + } + + makeCluster(out, null, currentClusterOffset, null, clusterSizes); + + long segmentSize = written - offsetSegmentSizeSet - 7; + + // final step write offsets and sizes + out.rewind(); + written = 0; + + skipTo(out, offsetSegmentSizeSet); + writeLong(out, segmentSize); + + if (predefinedDurations[durationFromTrackId] > -1) { + duration += predefinedDurations[durationFromTrackId];// this value is full-filled in makeTrackEntry() method + } + skipTo(out, offsetInfoDurationSet); + writeFloat(out, duration); + + firstClusterOffset -= baseSegmentOffset; + skipTo(out, offsetClusterSet); + writeInt(out, firstClusterOffset); + + skipTo(out, cueReservedOffset); + + /* Cue */ + dump(new byte[]{0x1c, 0x53, (byte) 0xbb, 0x6b, 0x20, 0x00, 0x00}, out); + + for (KeyFrame keyFrame : keyFrames) { + for (byte[] buffer : makeCuePoint(cuesForTrackId, keyFrame)) { + dump(buffer, out); + if (written >= (cueReservedOffset + 65535 - 16)) { + throw new IOException("Too many Cues"); + } + } + } + short cueSize = (short) (written - cueReservedOffset - 7); + + /* EBML Void */ + ByteBuffer voidBuffer = ByteBuffer.allocate(4); + voidBuffer.putShort((short) 0xec20); + voidBuffer.putShort((short) (firstClusterOffset - written - 4)); + dump(voidBuffer.array(), out); + + out.rewind(); + written = 0; + + skipTo(out, offsetCuesSet); + writeInt(out, (int) (cueReservedOffset - baseSegmentOffset)); + + skipTo(out, cueReservedOffset + 5); + writeShort(out, cueSize); + + for (int i = 0; i < clusterSizes.size(); i++) { + skipTo(out, clusterOffsets.get(i)); + byte[] size = ByteBuffer.allocate(4).putInt(clusterSizes.get(i) | 0x200000).array(); + out.write(size, 1, 3); + written += 3; + } + } + + private Block getNextBlockFrom(int internalTrackId) throws IOException { + if (readersSegment[internalTrackId] == null) { + readersSegment[internalTrackId] = readers[internalTrackId].getNextSegment(); + if (readersSegment[internalTrackId] == null) { + return null;// no more blocks in the selected track + } + } + + if (readersCluter[internalTrackId] == null) { + readersCluter[internalTrackId] = readersSegment[internalTrackId].getNextCluster(); + if (readersCluter[internalTrackId] == null) { + readersSegment[internalTrackId] = null; + return getNextBlockFrom(internalTrackId); + } + } + + SimpleBlock res = readersCluter[internalTrackId].getNextSimpleBlock(); + if (res == null) { + readersCluter[internalTrackId] = null; + return new Block();// fake block to indicate the end of the cluster + } + + Block bloq = new Block(); + bloq.data = res.data; + bloq.dataSize = (int) res.dataSize; + bloq.trackNumber = internalTrackId; + bloq.flags = res.flags; + bloq.absoluteTimecode = convertTimecode(res.relativeTimeCode, readersSegment[internalTrackId].info.timecodeScale, DEFAULT_TIMECODE_SCALE); + bloq.absoluteTimecode += readersCluter[internalTrackId].timecode; + + return bloq; + } + + private short convertTimecode(int time, long oldTimeScale, int newTimeScale) { + return (short) (time * (newTimeScale / oldTimeScale)); + } + + private void skipTo(SharpStream stream, long absoluteOffset) throws IOException { + absoluteOffset -= written; + written += absoluteOffset; + stream.skip(absoluteOffset); + } + + private void writeLong(SharpStream stream, long number) throws IOException { + byte[] buffer = ByteBuffer.allocate(DataReader.LONG_SIZE).putLong(number).array(); + stream.write(buffer, 1, buffer.length - 1); + written += buffer.length - 1; + } + + private void writeFloat(SharpStream stream, float number) throws IOException { + byte[] buffer = ByteBuffer.allocate(DataReader.FLOAT_SIZE).putFloat(number).array(); + dump(buffer, stream); + } + + private void writeShort(SharpStream stream, short number) throws IOException { + byte[] buffer = ByteBuffer.allocate(DataReader.SHORT_SIZE).putShort(number).array(); + dump(buffer, stream); + } + + private void writeInt(SharpStream stream, int number) throws IOException { + byte[] buffer = ByteBuffer.allocate(DataReader.INTEGER_SIZE).putInt(number).array(); + dump(buffer, stream); + } + + private void writeBlock(SharpStream stream, Block bloq, long clusterTimecode) throws IOException { + long relativeTimeCode = bloq.absoluteTimecode - clusterTimecode; + + if (relativeTimeCode < Short.MIN_VALUE || relativeTimeCode > Short.MAX_VALUE) { + throw new IndexOutOfBoundsException("SimpleBlock timecode overflow."); + } + + ArrayList listBuffer = new ArrayList<>(5); + listBuffer.add(new byte[]{(byte) 0xa3}); + listBuffer.add(null);// block size + listBuffer.add(encode(bloq.trackNumber + 1, false)); + listBuffer.add(ByteBuffer.allocate(DataReader.SHORT_SIZE).putShort((short) relativeTimeCode).array()); + listBuffer.add(new byte[]{bloq.flags}); + + int blockSize = bloq.dataSize; + for (int i = 2; i < listBuffer.size(); i++) { + blockSize += listBuffer.get(i).length; + } + listBuffer.set(1, encode(blockSize, false)); + + for (byte[] buff : listBuffer) { + dump(buff, stream); + } + + int read; + while ((read = bloq.data.read(outBuffer)) > 0) { + stream.write(outBuffer, 0, read); + written += read; + } + } + + private byte[] makeTimecode(long timecode) { + ByteBuffer buffer = ByteBuffer.allocate(9); + buffer.put((byte) 0xe7); + buffer.put(encode(timecode, true)); + + byte[] res = new byte[buffer.position()]; + System.arraycopy(buffer.array(), 0, res, 0, res.length); + + return res; + } + + private long makeCluster(SharpStream stream, byte[] bTimecode, long startOffset, ArrayList clusterOffsets, ArrayList clusterSizes) throws IOException { + if (startOffset > 0) { + clusterSizes.add((int) (written - startOffset));// size for last offset + } + + if (clusterOffsets != null) { + /* cluster */ + dump(new byte[]{0x1f, 0x43, (byte) 0xb6, 0x75}, stream); + clusterOffsets.add(written);// warning: max cluster size is 256 MiB + dump(new byte[]{0x20, 0x00, 0x00}, stream); + + startOffset = written;// size for the this cluster + + dump(bTimecode, stream); + + return startOffset; + } + + return -1; + } + + private void makeEBML(SharpStream stream) throws IOException { + // deafult values + dump(new byte[]{ + 0x1A, 0x45, (byte) 0xDF, (byte) 0xA3, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x1F, 0x42, (byte) 0x86, (byte) 0x81, 0x01, + 0x42, (byte) 0xF7, (byte) 0x81, 0x01, 0x42, (byte) 0xF2, (byte) 0x81, 0x04, + 0x42, (byte) 0xF3, (byte) 0x81, 0x08, 0x42, (byte) 0x82, (byte) 0x84, 0x77, + 0x65, 0x62, 0x6D, 0x42, (byte) 0x87, (byte) 0x81, 0x02, + 0x42, (byte) 0x85, (byte) 0x81, 0x02 + }, stream); + } + + private ArrayList makeTracks() { + ArrayList buffer = new ArrayList<>(1); + buffer.add(new byte[]{0x16, 0x54, (byte) 0xae, 0x6b}); + buffer.add(null); + + for (int i = 0; i < infoTracks.length; i++) { + buffer.addAll(makeTrackEntry(i, infoTracks[i])); + } + + return lengthFor(buffer); + } + + private ArrayList makeTrackEntry(int internalTrackId, WebMTrack track) { + byte[] id = encode(internalTrackId + 1, true); + ArrayList buffer = new ArrayList<>(12); + + /* track */ + buffer.add(new byte[]{(byte) 0xae}); + buffer.add(null); + + /* track number */ + buffer.add(new byte[]{(byte) 0xd7}); + buffer.add(id); + + /* track uid */ + buffer.add(new byte[]{0x73, (byte) 0xc5}); + buffer.add(id); + + /* flag lacing */ + buffer.add(new byte[]{(byte) 0x9c, (byte) 0x81, 0x00}); + + /* lang */ + buffer.add(new byte[]{0x22, (byte) 0xb5, (byte) 0x9c, (byte) 0x83, 0x75, 0x6e, 0x64}); + + /* codec id */ + buffer.add(new byte[]{(byte) 0x86}); + buffer.addAll(encode(track.codecId)); + + /* type */ + buffer.add(new byte[]{(byte) 0x83}); + buffer.add(encode(track.trackType, true)); + + /* default duration */ + if (track.defaultDuration != 0) { + predefinedDurations[internalTrackId] = (int) Math.ceil(track.defaultDuration / (float) DEFAULT_TIMECODE_SCALE); + buffer.add(new byte[]{0x23, (byte) 0xe3, (byte) 0x83}); + buffer.add(encode(track.defaultDuration, true)); + } + + /* audio/video */ + if ((track.trackType == 1 || track.trackType == 2) && valid(track.bMetadata)) { + buffer.add(new byte[]{(byte) (track.trackType == 1 ? 0xe0 : 0xe1)}); + buffer.add(encode(track.bMetadata.length, false)); + buffer.add(track.bMetadata); + } + + /* codec private*/ + if (valid(track.codecPrivate)) { + buffer.add(new byte[]{0x63, (byte) 0xa2}); + buffer.add(encode(track.codecPrivate.length, false)); + buffer.add(track.codecPrivate); + } + + return lengthFor(buffer); + + } + + private ArrayList makeCuePoint(int internalTrackId, KeyFrame keyFrame) { + ArrayList buffer = new ArrayList<>(5); + + /* CuePoint */ + buffer.add(new byte[]{(byte) 0xbb}); + buffer.add(null); + + /* CueTime */ + buffer.add(new byte[]{(byte) 0xb3}); + buffer.add(encode(keyFrame.atTimecode, true)); + + /* CueTrackPosition */ + buffer.addAll(makeCueTrackPosition(internalTrackId, keyFrame)); + + return lengthFor(buffer); + } + + private ArrayList makeCueTrackPosition(int internalTrackId, KeyFrame keyFrame) { + ArrayList buffer = new ArrayList<>(8); + + /* CueTrackPositions */ + buffer.add(new byte[]{(byte) 0xb7}); + buffer.add(null); + + /* CueTrack */ + buffer.add(new byte[]{(byte) 0xf7}); + buffer.add(encode(internalTrackId + 1, true)); + + /* CueClusterPosition */ + buffer.add(new byte[]{(byte) 0xf1}); + buffer.add(encode(keyFrame.atCluster, true)); + + /* CueRelativePosition */ + if (keyFrame.atBlock > 0) { + buffer.add(new byte[]{(byte) 0xf0}); + buffer.add(encode(keyFrame.atBlock, true)); + } + + return lengthFor(buffer); + } + + private void dump(byte[] buffer, SharpStream stream) throws IOException { + stream.write(buffer); + written += buffer.length; + } + + private ArrayList lengthFor(ArrayList buffer) { + long size = 0; + for (int i = 2; i < buffer.size(); i++) { + size += buffer.get(i).length; + } + buffer.set(1, encode(size, false)); + return buffer; + } + + private byte[] encode(long number, boolean withLength) { + int length = -1; + for (int i = 1; i <= 7; i++) { + if (number < Math.pow(2, 7 * i)) { + length = i; + break; + } + } + + if (length < 1) { + throw new ArithmeticException("Can't encode a number of bigger than 7 bytes"); + } + + if (number == (Math.pow(2, 7 * length)) - 1) { + length++; + } + + int offset = withLength ? 1 : 0; + byte[] buffer = new byte[offset + length]; + long marker = (long) Math.floor((length - 1) / 8); + + for (int i = length - 1, mul = 1; i >= 0; i--, mul *= 0x100) { + long b = (long) Math.floor(number / mul); + if (!withLength && i == marker) { + b = b | (0x80 >> (length - 1)); + } + buffer[offset + i] = (byte) b; + } + + if (withLength) { + buffer[0] = (byte) (0x80 | length); + } + + return buffer; + } + + private ArrayList encode(String value) { + byte[] str; + try { + str = value.getBytes("utf-8"); + } catch (UnsupportedEncodingException err) { + str = value.getBytes(); + } + + ArrayList buffer = new ArrayList<>(2); + buffer.add(encode(str.length, false)); + buffer.add(str); + + return buffer; + } + + private boolean valid(byte[] buffer) { + return buffer != null && buffer.length > 0; + } + + private int selectTrackForCue() { + int i = 0; + int videoTracks = 0; + int audioTracks = 0; + + for (; i < infoTracks.length; i++) { + switch (infoTracks[i].trackType) { + case 1: + videoTracks++; + break; + case 2: + audioTracks++; + break; + } + } + + int kind; + if (audioTracks == infoTracks.length) { + kind = 2; + } else if (videoTracks == infoTracks.length) { + kind = 1; + } else if (videoTracks > 0) { + kind = 1; + } else if (audioTracks > 0) { + kind = 2; + } else { + return 0; + } + + // TODO: in the adove code, find and select the shortest track for the desired kind + for (i = 0; i < infoTracks.length; i++) { + if (kind == infoTracks[i].trackType) { + return i; + } + } + + return 0; + } + + class KeyFrame { + + KeyFrame(long segment, long cluster, long block, int bTimecodeLength, long timecode) { + atCluster = cluster - segment; + if ((block - bTimecodeLength) > cluster) { + atBlock = (int) (block - cluster); + } + atTimecode = timecode; + } + + long atCluster; + int atBlock; + long atTimecode; + } + + class Block { + + InputStream data; + int trackNumber; + byte flags; + int dataSize; + long absoluteTimecode; + + boolean isKeyframe() { + return (flags & 0x80) == 0x80; + } + + @Override + public String toString() { + return String.format("trackNumber=%s isKeyFrame=%S absoluteTimecode=%s", trackNumber, (flags & 0x80) == 0x80, absoluteTimecode); + } + } +} diff --git a/app/src/main/java/org/schabi/newpipe/streams/io/SharpStream.java b/app/src/main/java/org/schabi/newpipe/streams/io/SharpStream.java new file mode 100644 index 000000000..48bea06f6 --- /dev/null +++ b/app/src/main/java/org/schabi/newpipe/streams/io/SharpStream.java @@ -0,0 +1,47 @@ +package org.schabi.newpipe.streams.io; + +import java.io.IOException; + +/** + * based c# + */ +public abstract class SharpStream { + + public abstract int read() throws IOException; + + public abstract int read(byte buffer[]) throws IOException; + + public abstract int read(byte buffer[], int offset, int count) throws IOException; + + public abstract long skip(long amount) throws IOException; + + + public abstract int available(); + + public abstract void rewind() throws IOException; + + + public abstract void dispose(); + + public abstract boolean isDisposed(); + + + public abstract boolean canRewind(); + + public abstract boolean canRead(); + + public abstract boolean canWrite(); + + + public abstract void write(byte value) throws IOException; + + public abstract void write(byte[] buffer) throws IOException; + + public abstract void write(byte[] buffer, int offset, int count) throws IOException; + + public abstract void flush() throws IOException; + + public void setLength(long length) throws IOException { + throw new IOException("Not implemented"); + } +} diff --git a/app/src/main/java/org/schabi/newpipe/util/SecondaryStreamHelper.java b/app/src/main/java/org/schabi/newpipe/util/SecondaryStreamHelper.java new file mode 100644 index 000000000..a5d3ea3eb --- /dev/null +++ b/app/src/main/java/org/schabi/newpipe/util/SecondaryStreamHelper.java @@ -0,0 +1,66 @@ +package org.schabi.newpipe.util; + +import android.support.annotation.NonNull; + +import org.schabi.newpipe.extractor.MediaFormat; +import org.schabi.newpipe.extractor.stream.AudioStream; +import org.schabi.newpipe.extractor.stream.Stream; +import org.schabi.newpipe.extractor.stream.VideoStream; +import org.schabi.newpipe.util.StreamItemAdapter.StreamSizeWrapper; + +import java.util.List; + +public class SecondaryStreamHelper { + private final int position; + private final StreamSizeWrapper streams; + + public SecondaryStreamHelper(StreamSizeWrapper streams, T selectedStream) { + this.streams = streams; + this.position = streams.getStreamsList().indexOf(selectedStream); + if (this.position < 0) throw new RuntimeException("selected stream not found"); + } + + public T getStream() { + return streams.getStreamsList().get(position); + } + + public long getSizeInBytes() { + return streams.getSizeInBytes(position); + } + + /** + * find the correct audio stream for the desired video stream + * + * @param audioStreams list of audio streams + * @param videoStream desired video ONLY stream + * @return selected audio stream or null if a candidate was not found + */ + public static AudioStream getAudioStreamFor(@NonNull List audioStreams, @NonNull VideoStream videoStream) { + // TODO: check if m4v and m4a selected streams are DASH compliant + switch (videoStream.getFormat()) { + case WEBM: + case MPEG_4: + break; + default: + return null; + } + + boolean m4v = videoStream.getFormat() == MediaFormat.MPEG_4; + + for (AudioStream audio : audioStreams) { + if (audio.getFormat() == (m4v ? MediaFormat.M4A : MediaFormat.WEBMA)) { + return audio; + } + } + + // retry, but this time in reverse order + for (int i = audioStreams.size() - 1; i >= 0; i--) { + AudioStream audio = audioStreams.get(i); + if (audio.getFormat() == (m4v ? MediaFormat.MP3 : MediaFormat.OPUS)) { + return audio; + } + } + + return null; + } +} diff --git a/app/src/main/java/org/schabi/newpipe/util/StreamItemAdapter.java b/app/src/main/java/org/schabi/newpipe/util/StreamItemAdapter.java index e100a447b..eb106f91d 100644 --- a/app/src/main/java/org/schabi/newpipe/util/StreamItemAdapter.java +++ b/app/src/main/java/org/schabi/newpipe/util/StreamItemAdapter.java @@ -1,6 +1,7 @@ package org.schabi.newpipe.util; import android.content.Context; +import android.util.SparseArray; import android.view.LayoutInflater; import android.view.View; import android.view.ViewGroup; @@ -13,6 +14,7 @@ import org.schabi.newpipe.Downloader; import org.schabi.newpipe.R; import org.schabi.newpipe.extractor.stream.AudioStream; import org.schabi.newpipe.extractor.stream.Stream; +import org.schabi.newpipe.extractor.stream.SubtitlesStream; import org.schabi.newpipe.extractor.stream.VideoStream; import java.io.Serializable; @@ -28,26 +30,34 @@ import us.shandian.giga.util.Utility; /** * A list adapter for a list of {@link Stream streams}, currently supporting {@link VideoStream} and {@link AudioStream}. */ -public class StreamItemAdapter extends BaseAdapter { +public class StreamItemAdapter extends BaseAdapter { private final Context context; private final StreamSizeWrapper streamsWrapper; - private final boolean showIconNoAudio; + private final SparseArray> secondaryStreams; - public StreamItemAdapter(Context context, StreamSizeWrapper streamsWrapper, boolean showIconNoAudio) { + public StreamItemAdapter(Context context, StreamSizeWrapper streamsWrapper, SparseArray> secondaryStreams) { this.context = context; this.streamsWrapper = streamsWrapper; - this.showIconNoAudio = showIconNoAudio; + this.secondaryStreams = secondaryStreams; + } + + public StreamItemAdapter(Context context, StreamSizeWrapper streamsWrapper, boolean showIconNoAudio) { + this(context, streamsWrapper, showIconNoAudio ? new SparseArray<>() : null); } public StreamItemAdapter(Context context, StreamSizeWrapper streamsWrapper) { - this(context, streamsWrapper, false); + this(context, streamsWrapper, null); } public List getAll() { return streamsWrapper.getStreamsList(); } + public SparseArray> getAllSecondary() { + return secondaryStreams; + } + @Override public int getCount() { return streamsWrapper.getStreamsList().size(); @@ -89,29 +99,46 @@ public class StreamItemAdapter extends BaseAdapter { String qualityString; if (stream instanceof VideoStream) { - qualityString = ((VideoStream) stream).getResolution(); + VideoStream videoStream = ((VideoStream) stream); + qualityString = videoStream.getResolution(); - if (!showIconNoAudio) { - woSoundIconVisibility = View.GONE; - } else if (((VideoStream) stream).isVideoOnly()) { - woSoundIconVisibility = View.VISIBLE; - } else if (isDropdownItem) { - woSoundIconVisibility = View.INVISIBLE; + if (secondaryStreams != null) { + if (videoStream.isVideoOnly()) { + woSoundIconVisibility = secondaryStreams.get(position) == null ? View.VISIBLE : View.INVISIBLE; + } else if (isDropdownItem) { + woSoundIconVisibility = View.INVISIBLE; + } } } else if (stream instanceof AudioStream) { qualityString = ((AudioStream) stream).getAverageBitrate() + "kbps"; + } else if (stream instanceof SubtitlesStream) { + qualityString = ((SubtitlesStream) stream).getDisplayLanguageName(); + if (((SubtitlesStream) stream).isAutoGenerated()) { + qualityString += " (" + context.getString(R.string.caption_auto_generated) + ")"; + } } else { qualityString = stream.getFormat().getSuffix(); } if (streamsWrapper.getSizeInBytes(position) > 0) { - sizeView.setText(streamsWrapper.getFormattedSize(position)); + SecondaryStreamHelper secondary = secondaryStreams == null ? null : secondaryStreams.get(position); + if (secondary != null) { + long size = secondary.getSizeInBytes() + streamsWrapper.getSizeInBytes(position); + sizeView.setText(Utility.formatBytes(size)); + } else { + sizeView.setText(streamsWrapper.getFormattedSize(position)); + } sizeView.setVisibility(View.VISIBLE); } else { sizeView.setVisibility(View.GONE); } - formatNameView.setText(stream.getFormat().getName()); + if (stream instanceof SubtitlesStream) { + formatNameView.setText(((SubtitlesStream) stream).getLanguageTag()); + } else { + formatNameView.setText(stream.getFormat().getName()); + } + qualityView.setText(qualityString); woSoundIconView.setVisibility(woSoundIconVisibility); @@ -122,15 +149,17 @@ public class StreamItemAdapter extends BaseAdapter { * A wrapper class that includes a way of storing the stream sizes. */ public static class StreamSizeWrapper implements Serializable { - private static final StreamSizeWrapper EMPTY = new StreamSizeWrapper<>(Collections.emptyList()); + private static final StreamSizeWrapper EMPTY = new StreamSizeWrapper<>(Collections.emptyList(), null); private final List streamsList; private final long[] streamSizes; + private final String unknownSize; - public StreamSizeWrapper(List streamsList) { + public StreamSizeWrapper(List streamsList, Context context) { this.streamsList = streamsList; this.streamSizes = new long[streamsList.size()]; + this.unknownSize = context == null ? "--.-" : context.getString(R.string.unknown_content); - for (int i = 0; i < streamSizes.length; i++) streamSizes[i] = -1; + for (int i = 0; i < streamSizes.length; i++) streamSizes[i] = -2; } /** @@ -143,7 +172,7 @@ public class StreamItemAdapter extends BaseAdapter { final Callable fetchAndSet = () -> { boolean hasChanged = false; for (X stream : streamsWrapper.getStreamsList()) { - if (streamsWrapper.getSizeInBytes(stream) > 0) { + if (streamsWrapper.getSizeInBytes(stream) > -2) { continue; } @@ -173,11 +202,18 @@ public class StreamItemAdapter extends BaseAdapter { } public String getFormattedSize(int streamIndex) { - return Utility.formatBytes(getSizeInBytes(streamIndex)); + return formatSize(getSizeInBytes(streamIndex)); } public String getFormattedSize(T stream) { - return Utility.formatBytes(getSizeInBytes(stream)); + return formatSize(getSizeInBytes(stream)); + } + + private String formatSize(long size) { + if (size > -1) { + return Utility.formatBytes(size); + } + return unknownSize; } public void setSize(int streamIndex, long sizeInBytes) { @@ -193,4 +229,4 @@ public class StreamItemAdapter extends BaseAdapter { return (StreamSizeWrapper) EMPTY; } } -} \ No newline at end of file +} diff --git a/app/src/main/java/us/shandian/giga/get/DownloadDataSource.java b/app/src/main/java/us/shandian/giga/get/DownloadDataSource.java deleted file mode 100644 index 2a8a9e129..000000000 --- a/app/src/main/java/us/shandian/giga/get/DownloadDataSource.java +++ /dev/null @@ -1,40 +0,0 @@ -package us.shandian.giga.get; - -import java.util.List; - -/** - * Provides access to the storage of {@link DownloadMission}s - */ -public interface DownloadDataSource { - - /** - * Load all missions - * - * @return a list of download missions - */ - List loadMissions(); - - /** - * Add a download mission to the storage - * - * @param downloadMission the download mission to add - * @return the identifier of the mission - */ - void addMission(DownloadMission downloadMission); - - /** - * Update a download mission which exists in the storage - * - * @param downloadMission the download mission to update - * @throws IllegalArgumentException if the mission was not added to storage - */ - void updateMission(DownloadMission downloadMission); - - - /** - * Delete a download mission - * - * @param downloadMission the mission to delete - */ - void deleteMission(DownloadMission downloadMission); -} \ No newline at end of file diff --git a/app/src/main/java/us/shandian/giga/get/DownloadInitializer.java b/app/src/main/java/us/shandian/giga/get/DownloadInitializer.java new file mode 100644 index 000000000..ce7ae267c --- /dev/null +++ b/app/src/main/java/us/shandian/giga/get/DownloadInitializer.java @@ -0,0 +1,186 @@ +package us.shandian.giga.get; + +import android.support.annotation.NonNull; +import android.util.Log; + +import java.io.File; +import java.io.IOException; +import java.io.InterruptedIOException; +import java.io.RandomAccessFile; +import java.net.HttpURLConnection; +import java.nio.channels.ClosedByInterruptException; + +import us.shandian.giga.util.Utility; + +import static org.schabi.newpipe.BuildConfig.DEBUG; + +public class DownloadInitializer extends Thread { + private final static String TAG = "DownloadInitializer"; + final static int mId = 0; + + private DownloadMission mMission; + private HttpURLConnection mConn; + + DownloadInitializer(@NonNull DownloadMission mission) { + mMission = mission; + mConn = null; + } + + @Override + public void run() { + if (mMission.current > 0) mMission.resetState(); + + int retryCount = 0; + while (true) { + try { + mMission.currentThreadCount = mMission.threadCount; + + mConn = mMission.openConnection(mId, -1, -1); + mMission.establishConnection(mId, mConn); + + if (!mMission.running || Thread.interrupted()) return; + + mMission.length = Utility.getContentLength(mConn); + + + if (mMission.length == 0) { + mMission.notifyError(DownloadMission.ERROR_HTTP_NO_CONTENT, null); + return; + } + + // check for dynamic generated content + if (mMission.length == -1 && mConn.getResponseCode() == 200) { + mMission.blocks = 0; + mMission.length = 0; + mMission.fallback = true; + mMission.unknownLength = true; + mMission.currentThreadCount = 1; + + if (DEBUG) { + Log.d(TAG, "falling back (unknown length)"); + } + } else { + // Open again + mConn = mMission.openConnection(mId, mMission.length - 10, mMission.length); + mMission.establishConnection(mId, mConn); + + if (!mMission.running || Thread.interrupted()) return; + + synchronized (mMission.blockState) { + if (mConn.getResponseCode() == 206) { + if (mMission.currentThreadCount > 1) { + mMission.blocks = mMission.length / DownloadMission.BLOCK_SIZE; + + if (mMission.currentThreadCount > mMission.blocks) { + mMission.currentThreadCount = (int) mMission.blocks; + } + if (mMission.currentThreadCount <= 0) { + mMission.currentThreadCount = 1; + } + if (mMission.blocks * DownloadMission.BLOCK_SIZE < mMission.length) { + mMission.blocks++; + } + } else { + // if one thread is solicited don't calculate blocks, is useless + mMission.blocks = 1; + mMission.fallback = true; + mMission.unknownLength = false; + } + + if (DEBUG) { + Log.d(TAG, "http response code = " + mConn.getResponseCode()); + } + } else { + // Fallback to single thread + mMission.blocks = 0; + mMission.fallback = true; + mMission.unknownLength = false; + mMission.currentThreadCount = 1; + + if (DEBUG) { + Log.d(TAG, "falling back due http response code = " + mConn.getResponseCode()); + } + } + + for (long i = 0; i < mMission.currentThreadCount; i++) { + mMission.threadBlockPositions.add(i); + mMission.threadBytePositions.add(0L); + } + } + + if (!mMission.running || Thread.interrupted()) return; + } + + File file; + if (mMission.current == 0) { + file = new File(mMission.location); + if (!Utility.mkdir(file, true)) { + mMission.notifyError(DownloadMission.ERROR_PATH_CREATION, null); + return; + } + + file = new File(file, mMission.name); + + // if the name is used by another process, delete it + if (file.exists() && !file.isFile() && !file.delete()) { + mMission.notifyError(DownloadMission.ERROR_FILE_CREATION, null); + return; + } + + if (!file.exists() && !file.createNewFile()) { + mMission.notifyError(DownloadMission.ERROR_FILE_CREATION, null); + return; + } + } else { + file = new File(mMission.location, mMission.name); + } + + RandomAccessFile af = new RandomAccessFile(file, "rw"); + af.setLength(mMission.offsets[mMission.current] + mMission.length); + af.seek(mMission.offsets[mMission.current]); + af.close(); + + if (!mMission.running || Thread.interrupted()) return; + + mMission.running = false; + break; + } catch (InterruptedIOException | ClosedByInterruptException e) { + return; + } catch (Exception e) { + if (!mMission.running) return; + + if (e instanceof IOException && e.getMessage().contains("Permission denied")) { + mMission.notifyError(DownloadMission.ERROR_PERMISSION_DENIED, e); + return; + } + + if (retryCount++ > mMission.maxRetry) { + Log.e(TAG, "initializer failed", e); + mMission.running = false; + mMission.notifyError(e); + return; + } + + Log.e(TAG, "initializer failed, retrying", e); + } + } + + // hide marquee in the progress bar + mMission.done++; + + mMission.start(); + } + + @Override + public void interrupt() { + super.interrupt(); + + if (mConn != null) { + try { + mConn.disconnect(); + } catch (Exception e) { + // nothing to do + } + } + } +} diff --git a/app/src/main/java/us/shandian/giga/get/DownloadManager.java b/app/src/main/java/us/shandian/giga/get/DownloadManager.java deleted file mode 100644 index 45beb5563..000000000 --- a/app/src/main/java/us/shandian/giga/get/DownloadManager.java +++ /dev/null @@ -1,53 +0,0 @@ -package us.shandian.giga.get; - -public interface DownloadManager { - int BLOCK_SIZE = 512 * 1024; - - /** - * Start a new download mission - * - * @param url the url to download - * @param location the location - * @param name the name of the file to create - * @param isAudio true if the download is an audio file - * @param threads the number of threads maximal used to download chunks of the file. @return the identifier of the mission. - */ - int startMission(String url, String location, String name, boolean isAudio, int threads); - - /** - * Resume the execution of a download mission. - * - * @param id the identifier of the mission to resume. - */ - void resumeMission(int id); - - /** - * Pause the execution of a download mission. - * - * @param id the identifier of the mission to pause. - */ - void pauseMission(int id); - - /** - * Deletes the mission from the downloaded list but keeps the downloaded file. - * - * @param id The mission identifier - */ - void deleteMission(int id); - - /** - * Get the download mission by its identifier - * - * @param id the identifier of the download mission - * @return the download mission or null if the mission doesn't exist - */ - DownloadMission getMission(int id); - - /** - * Get the number of download missions. - * - * @return the number of download missions. - */ - int getCount(); - -} diff --git a/app/src/main/java/us/shandian/giga/get/DownloadManagerImpl.java b/app/src/main/java/us/shandian/giga/get/DownloadManagerImpl.java deleted file mode 100755 index a377d861c..000000000 --- a/app/src/main/java/us/shandian/giga/get/DownloadManagerImpl.java +++ /dev/null @@ -1,395 +0,0 @@ -package us.shandian.giga.get; - -import android.content.Context; -import android.content.Intent; -import android.os.Handler; -import android.support.annotation.NonNull; -import android.support.annotation.Nullable; -import android.util.Log; - -import org.schabi.newpipe.download.ExtSDDownloadFailedActivity; - -import java.io.File; -import java.io.FilenameFilter; -import java.io.IOException; -import java.io.RandomAccessFile; -import java.net.HttpURLConnection; -import java.net.URL; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.List; - -import us.shandian.giga.util.Utility; - -import static org.schabi.newpipe.BuildConfig.DEBUG; - -public class DownloadManagerImpl implements DownloadManager { - private static final String TAG = DownloadManagerImpl.class.getSimpleName(); - private final DownloadDataSource mDownloadDataSource; - - private final ArrayList mMissions = new ArrayList<>(); - @NonNull - private final Context context; - - /** - * Create a new instance - * - * @param searchLocations the directories to search for unfinished downloads - * @param downloadDataSource the data source for finished downloads - */ - public DownloadManagerImpl(Collection searchLocations, DownloadDataSource downloadDataSource) { - mDownloadDataSource = downloadDataSource; - this.context = null; - loadMissions(searchLocations); - } - - public DownloadManagerImpl(Collection searchLocations, DownloadDataSource downloadDataSource, Context context) { - mDownloadDataSource = downloadDataSource; - this.context = context; - loadMissions(searchLocations); - } - - @Override - public int startMission(String url, String location, String name, boolean isAudio, int threads) { - DownloadMission existingMission = getMissionByLocation(location, name); - if (existingMission != null) { - // Already downloaded or downloading - if (existingMission.finished) { - // Overwrite mission - deleteMission(mMissions.indexOf(existingMission)); - } else { - // Rename file (?) - try { - name = generateUniqueName(location, name); - } catch (Exception e) { - Log.e(TAG, "Unable to generate unique name", e); - name = System.currentTimeMillis() + name; - Log.i(TAG, "Using " + name); - } - } - } - - DownloadMission mission = new DownloadMission(name, url, location); - mission.timestamp = System.currentTimeMillis(); - mission.threadCount = threads; - mission.addListener(new MissionListener(mission)); - new Initializer(mission).start(); - return insertMission(mission); - } - - @Override - public void resumeMission(int i) { - DownloadMission d = getMission(i); - if (!d.running && d.errCode == -1) { - d.start(); - } - } - - @Override - public void pauseMission(int i) { - DownloadMission d = getMission(i); - if (d.running) { - d.pause(); - } - } - - @Override - public void deleteMission(int i) { - DownloadMission mission = getMission(i); - if (mission.finished) { - mDownloadDataSource.deleteMission(mission); - } - mission.delete(); - mMissions.remove(i); - } - - private void loadMissions(Iterable searchLocations) { - mMissions.clear(); - loadFinishedMissions(); - for (String location : searchLocations) { - loadMissions(location); - } - - } - - /** - * Sort a list of mission by its timestamp. Oldest first - * @param missions the missions to sort - */ - static void sortByTimestamp(List missions) { - Collections.sort(missions, new Comparator() { - @Override - public int compare(DownloadMission o1, DownloadMission o2) { - return Long.compare(o1.timestamp, o2.timestamp); - } - }); - } - - /** - * Loads finished missions from the data source - */ - private void loadFinishedMissions() { - List finishedMissions = mDownloadDataSource.loadMissions(); - if (finishedMissions == null) { - finishedMissions = new ArrayList<>(); - } - // Ensure its sorted - sortByTimestamp(finishedMissions); - - mMissions.ensureCapacity(mMissions.size() + finishedMissions.size()); - for (DownloadMission mission : finishedMissions) { - File downloadedFile = mission.getDownloadedFile(); - if (!downloadedFile.isFile()) { - if (DEBUG) { - Log.d(TAG, "downloaded file removed: " + downloadedFile.getAbsolutePath()); - } - mDownloadDataSource.deleteMission(mission); - } else { - mission.length = downloadedFile.length(); - mission.finished = true; - mission.running = false; - mMissions.add(mission); - } - } - } - - private void loadMissions(String location) { - - File f = new File(location); - - if (f.exists() && f.isDirectory()) { - File[] subs = f.listFiles(); - - if (subs == null) { - Log.e(TAG, "listFiles() returned null"); - return; - } - - for (File sub : subs) { - if (sub.isFile() && sub.getName().endsWith(".giga")) { - DownloadMission mis = Utility.readFromFile(sub.getAbsolutePath()); - if (mis != null) { - if (mis.finished) { - if (!sub.delete()) { - Log.w(TAG, "Unable to delete .giga file: " + sub.getPath()); - } - continue; - } - - mis.running = false; - mis.recovered = true; - insertMission(mis); - } - } - } - } - } - - @Override - public DownloadMission getMission(int i) { - return mMissions.get(i); - } - - @Override - public int getCount() { - return mMissions.size(); - } - - private int insertMission(DownloadMission mission) { - int i = -1; - - DownloadMission m = null; - - if (mMissions.size() > 0) { - do { - m = mMissions.get(++i); - } while (m.timestamp > mission.timestamp && i < mMissions.size() - 1); - - //if (i > 0) i--; - } else { - i = 0; - } - - mMissions.add(i, mission); - - return i; - } - - /** - * Get a mission by its location and name - * - * @param location the location - * @param name the name - * @return the mission or null if no such mission exists - */ - private - @Nullable - DownloadMission getMissionByLocation(String location, String name) { - for (DownloadMission mission : mMissions) { - if (location.equals(mission.location) && name.equals(mission.name)) { - return mission; - } - } - return null; - } - - /** - * Splits the filename into name and extension - *

- * Dots are ignored if they appear: not at all, at the beginning of the file, - * at the end of the file - * - * @param name the name to split - * @return a string array with a length of 2 containing the name and the extension - */ - private static String[] splitName(String name) { - int dotIndex = name.lastIndexOf('.'); - if (dotIndex <= 0 || (dotIndex == name.length() - 1)) { - return new String[]{name, ""}; - } else { - return new String[]{name.substring(0, dotIndex), name.substring(dotIndex + 1)}; - } - } - - /** - * Generates a unique file name. - *

- * e.g. "myname (1).txt" if the name "myname.txt" exists. - * - * @param location the location (to check for existing files) - * @param name the name of the file - * @return the unique file name - * @throws IllegalArgumentException if the location is not a directory - * @throws SecurityException if the location is not readable - */ - private static String generateUniqueName(String location, String name) { - if (location == null) throw new NullPointerException("location is null"); - if (name == null) throw new NullPointerException("name is null"); - File destination = new File(location); - if (!destination.isDirectory()) { - throw new IllegalArgumentException("location is not a directory: " + location); - } - final String[] nameParts = splitName(name); - String[] existingName = destination.list(new FilenameFilter() { - @Override - public boolean accept(File dir, String name) { - return name.startsWith(nameParts[0]); - } - }); - Arrays.sort(existingName); - String newName; - int downloadIndex = 0; - do { - newName = nameParts[0] + " (" + downloadIndex + ")." + nameParts[1]; - ++downloadIndex; - if (downloadIndex == 1000) { // Probably an error on our side - throw new RuntimeException("Too many existing files"); - } - } while (Arrays.binarySearch(existingName, newName) >= 0); - return newName; - } - - private class Initializer extends Thread { - private final DownloadMission mission; - private final Handler handler; - - public Initializer(DownloadMission mission) { - this.mission = mission; - this.handler = new Handler(); - } - - @Override - public void run() { - try { - URL url = new URL(mission.url); - HttpURLConnection conn = (HttpURLConnection) url.openConnection(); - mission.length = conn.getContentLength(); - - if (mission.length <= 0) { - mission.errCode = DownloadMission.ERROR_SERVER_UNSUPPORTED; - //mission.notifyError(DownloadMission.ERROR_SERVER_UNSUPPORTED); - return; - } - - // Open again - conn = (HttpURLConnection) url.openConnection(); - conn.setRequestProperty("Range", "bytes=" + (mission.length - 10) + "-" + mission.length); - - if (conn.getResponseCode() != 206) { - // Fallback to single thread if no partial content support - mission.fallback = true; - - if (DEBUG) { - Log.d(TAG, "falling back"); - } - } - - if (DEBUG) { - Log.d(TAG, "response = " + conn.getResponseCode()); - } - - mission.blocks = mission.length / BLOCK_SIZE; - - if (mission.threadCount > mission.blocks) { - mission.threadCount = (int) mission.blocks; - } - - if (mission.threadCount <= 0) { - mission.threadCount = 1; - } - - if (mission.blocks * BLOCK_SIZE < mission.length) { - mission.blocks++; - } - - - new File(mission.location).mkdirs(); - new File(mission.location + "/" + mission.name).createNewFile(); - RandomAccessFile af = new RandomAccessFile(mission.location + "/" + mission.name, "rw"); - af.setLength(mission.length); - af.close(); - - mission.start(); - } catch (IOException ie) { - if(context == null) throw new RuntimeException(ie); - - if(ie.getMessage().contains("Permission denied")) { - handler.post(() -> - context.startActivity(new Intent(context, ExtSDDownloadFailedActivity.class))); - } else throw new RuntimeException(ie); - } catch (Exception e) { - // TODO Notify - throw new RuntimeException(e); - } - } - } - - /** - * Waits for mission to finish to add it to the {@link #mDownloadDataSource} - */ - private class MissionListener implements DownloadMission.MissionListener { - private final DownloadMission mMission; - - private MissionListener(DownloadMission mission) { - if (mission == null) throw new NullPointerException("mission is null"); - // Could the mission be passed in onFinish()? - mMission = mission; - } - - @Override - public void onProgressUpdate(DownloadMission downloadMission, long done, long total) { - } - - @Override - public void onFinish(DownloadMission downloadMission) { - mDownloadDataSource.addMission(mMission); - } - - @Override - public void onError(DownloadMission downloadMission, int errCode) { - } - } -} diff --git a/app/src/main/java/us/shandian/giga/get/DownloadMission.java b/app/src/main/java/us/shandian/giga/get/DownloadMission.java index 79c4baf05..c25d517f1 100644 --- a/app/src/main/java/us/shandian/giga/get/DownloadMission.java +++ b/app/src/main/java/us/shandian/giga/get/DownloadMission.java @@ -1,102 +1,170 @@ package us.shandian.giga.get; import android.os.Handler; -import android.os.Looper; +import android.os.Message; import android.util.Log; import java.io.File; -import java.io.ObjectInputStream; -import java.io.Serializable; -import java.lang.ref.WeakReference; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.net.ConnectException; +import java.net.HttpURLConnection; +import java.net.URL; +import java.net.UnknownHostException; import java.util.ArrayList; import java.util.HashMap; -import java.util.Iterator; import java.util.List; -import java.util.Map; +import javax.net.ssl.SSLException; + +import us.shandian.giga.postprocessing.Postprocessing; +import us.shandian.giga.service.DownloadManagerService; import us.shandian.giga.util.Utility; import static org.schabi.newpipe.BuildConfig.DEBUG; -public class DownloadMission implements Serializable { - private static final long serialVersionUID = 0L; +public class DownloadMission extends Mission { + private static final long serialVersionUID = 3L;// last bump: 8 november 2018 - private static final String TAG = DownloadMission.class.getSimpleName(); + static final int BUFFER_SIZE = 64 * 1024; + final static int BLOCK_SIZE = 512 * 1024; - public interface MissionListener { - HashMap handlerStore = new HashMap<>(); + private static final String TAG = "DownloadMission"; - void onProgressUpdate(DownloadMission downloadMission, long done, long total); - - void onFinish(DownloadMission downloadMission); - - void onError(DownloadMission downloadMission, int errCode); - } - - public static final int ERROR_SERVER_UNSUPPORTED = 206; - public static final int ERROR_UNKNOWN = 233; + public static final int ERROR_NOTHING = -1; + public static final int ERROR_PATH_CREATION = 1000; + public static final int ERROR_FILE_CREATION = 1001; + public static final int ERROR_UNKNOWN_EXCEPTION = 1002; + public static final int ERROR_PERMISSION_DENIED = 1003; + public static final int ERROR_SSL_EXCEPTION = 1004; + public static final int ERROR_UNKNOWN_HOST = 1005; + public static final int ERROR_CONNECT_HOST = 1006; + public static final int ERROR_POSTPROCESSING_FAILED = 1007; + public static final int ERROR_HTTP_NO_CONTENT = 204; + public static final int ERROR_HTTP_UNSUPPORTED_RANGE = 206; /** - * The filename + * The urls of the file to download */ - public String name; + public String[] urls; /** - * The url of the file to download + * Number of blocks the size of {@link DownloadMission#BLOCK_SIZE} */ - public String url; - - /** - * The directory to store the download - */ - public String location; - - /** - * Number of blocks the size of {@link DownloadManager#BLOCK_SIZE} - */ - public long blocks; - - /** - * Number of bytes - */ - public long length; + long blocks = -1; /** * Number of bytes downloaded */ public long done; + + /** + * Indicates a file generated dynamically on the web server + */ + public boolean unknownLength; + + /** + * offset in the file where the data should be written + */ + public long[] offsets; + + /** + * The post-processing algorithm arguments + */ + public String[] postprocessingArgs; + + /** + * The post-processing algorithm name + */ + public String postprocessingName; + + /** + * Indicates if the post-processing algorithm is actually running, used to detect corrupt downloads + */ + public boolean postprocessingRunning; + + /** + * Indicate if the post-processing algorithm works on the same file + */ + public boolean postprocessingThis; + + /** + * The current resource to download {@code urls[current]} + */ + public int current; + + /** + * Metadata where the mission state is saved + */ + public File metadata; + + /** + * maximum attempts + */ + public int maxRetry; + + /** + * Approximated final length, this represent the sum of all resources sizes + */ + public long nearLength; + public int threadCount = 3; - public int finishCount; - private final List threadPositions = new ArrayList<>(); - public final Map blockState = new HashMap<>(); - public boolean running; - public boolean finished; - public boolean fallback; - public int errCode = -1; - public long timestamp; + boolean fallback; + private int finishCount; + public transient boolean running; + public transient boolean enqueued = true; + public int errCode = ERROR_NOTHING; + + public transient Exception errObject = null; public transient boolean recovered; - - private transient ArrayList> mListeners = new ArrayList<>(); + public transient Handler mHandler; private transient boolean mWritingToFile; - private static final int NO_IDENTIFIER = -1; + @SuppressWarnings("UseSparseArrays")// LongSparseArray is not serializable + final HashMap blockState = new HashMap<>(); + final List threadBlockPositions = new ArrayList<>(); + final List threadBytePositions = new ArrayList<>(); + + private transient boolean deleted; + int currentThreadCount; + private transient Thread[] threads = new Thread[0]; + private transient Thread init = null; + + + protected DownloadMission() { - public DownloadMission() { } - public DownloadMission(String name, String url, String location) { + public DownloadMission(String url, String name, String location, char kind) { + this(new String[]{url}, name, location, kind, null, null); + } + + public DownloadMission(String[] urls, String name, String location, char kind, String postprocessingName, String[] postprocessingArgs) { if (name == null) throw new NullPointerException("name is null"); if (name.isEmpty()) throw new IllegalArgumentException("name is empty"); - if (url == null) throw new NullPointerException("url is null"); - if (url.isEmpty()) throw new IllegalArgumentException("url is empty"); + if (urls == null) throw new NullPointerException("urls is null"); + if (urls.length < 1) throw new IllegalArgumentException("urls is empty"); if (location == null) throw new NullPointerException("location is null"); if (location.isEmpty()) throw new IllegalArgumentException("location is empty"); - this.url = url; + this.urls = urls; this.name = name; this.location = location; - } + this.kind = kind; + this.offsets = new long[urls.length]; + if (postprocessingName != null) { + Postprocessing algorithm = Postprocessing.getAlgorithm(postprocessingName, null); + this.postprocessingThis = algorithm.worksOnSameFile; + this.offsets[0] = algorithm.recommendedReserve; + this.postprocessingName = postprocessingName; + this.postprocessingArgs = postprocessingArgs; + } else { + if (DEBUG && urls.length > 1) { + Log.w(TAG, "mission created with multiple urls ¿missing post-processing algorithm?"); + } + } + } private void checkBlock(long block) { if (block < 0 || block >= blocks) { @@ -110,12 +178,12 @@ public class DownloadMission implements Serializable { * @param block the block identifier * @return true if the block is reserved and false if otherwise */ - public boolean isBlockPreserved(long block) { + boolean isBlockPreserved(long block) { checkBlock(block); return blockState.containsKey(block) ? blockState.get(block) : false; } - public void preserveBlock(long block) { + void preserveBlock(long block) { checkBlock(block); synchronized (blockState) { blockState.put(block, true); @@ -123,125 +191,211 @@ public class DownloadMission implements Serializable { } /** - * Set the download position of the file + * Set the block of the file * * @param threadId the identifier of the thread - * @param position the download position of the thread + * @param position the block of the thread */ - public void setPosition(int threadId, long position) { - threadPositions.set(threadId, position); + void setBlockPosition(int threadId, long position) { + threadBlockPositions.set(threadId, position); } /** - * Get the position of a thread + * Get the block of a file * * @param threadId the identifier of the thread - * @return the position for the thread + * @return the block for the thread */ - public long getPosition(int threadId) { - return threadPositions.get(threadId); + long getBlockPosition(int threadId) { + return threadBlockPositions.get(threadId); } - public synchronized void notifyProgress(long deltaLen) { + /** + * Save the position of the desired thread + * + * @param threadId the identifier of the thread + * @param position the relative position in bytes or zero + */ + void setThreadBytePosition(int threadId, long position) { + threadBytePositions.set(threadId, position); + } + + /** + * Get position inside of the thread, where thread will be resumed + * + * @param threadId the identifier of the thread + * @return the relative position in bytes or zero + */ + long getThreadBytePosition(int threadId) { + return threadBytePositions.get(threadId); + } + + /** + * Open connection + * + * @param threadId id of the calling thread, used only for debug + * @param rangeStart range start + * @param rangeEnd range end + * @return a {@link java.net.URLConnection URLConnection} linking to the URL. + * @throws IOException if an I/O exception occurs. + */ + HttpURLConnection openConnection(int threadId, long rangeStart, long rangeEnd) throws IOException { + URL url = new URL(urls[current]); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setInstanceFollowRedirects(true); + + if (rangeStart >= 0) { + String req = "bytes=" + rangeStart + "-"; + if (rangeEnd > 0) req += rangeEnd; + + conn.setRequestProperty("Range", req); + + if (DEBUG) { + Log.d(TAG, threadId + ":" + conn.getRequestProperty("Range")); + } + } + + return conn; + } + + /** + * @param threadId id of the calling thread + * @param conn Opens and establish the communication + * @throws IOException if an error occurred connecting to the server. + * @throws HttpError if the HTTP Status-Code is not satisfiable + */ + void establishConnection(int threadId, HttpURLConnection conn) throws IOException, HttpError { + conn.connect(); + int statusCode = conn.getResponseCode(); + + if (DEBUG) { + Log.d(TAG, threadId + ":Content-Length=" + conn.getContentLength() + " Code:" + statusCode); + } + + switch (statusCode) { + case 204: + case 205: + case 207: + throw new HttpError(conn.getResponseCode()); + case 416: + return;// let the download thread handle this error + default: + if (statusCode < 200 || statusCode > 299) { + throw new HttpError(statusCode); + } + } + + } + + + private void notify(int what) { + Message m = new Message(); + m.what = what; + m.obj = this; + + mHandler.sendMessage(m); + } + + synchronized void notifyProgress(long deltaLen) { if (!running) return; if (recovered) { recovered = false; } + if (unknownLength) { + length += deltaLen;// Update length before proceeding + } + done += deltaLen; if (done > length) { done = length; } - if (done != length) { - writeThisToFile(); + if (done != length && !deleted && !mWritingToFile) { + mWritingToFile = true; + runAsync(-2, this::writeThisToFile); } - for (WeakReference ref : mListeners) { - final MissionListener listener = ref.get(); - if (listener != null) { - MissionListener.handlerStore.get(listener).post(new Runnable() { - @Override - public void run() { - listener.onProgressUpdate(DownloadMission.this, done, length); - } - }); - } + notify(DownloadManagerService.MESSAGE_PROGRESS); + } + + synchronized void notifyError(Exception err) { + Log.e(TAG, "notifyError()", err); + + if (err instanceof FileNotFoundException) { + notifyError(ERROR_FILE_CREATION, null); + } else if (err instanceof SSLException) { + notifyError(ERROR_SSL_EXCEPTION, null); + } else if (err instanceof HttpError) { + notifyError(((HttpError) err).statusCode, null); + } else if (err instanceof ConnectException) { + notifyError(ERROR_CONNECT_HOST, null); + } else if (err instanceof UnknownHostException) { + notifyError(ERROR_UNKNOWN_HOST, null); + } else { + notifyError(ERROR_UNKNOWN_EXCEPTION, err); } } - /** - * Called by a download thread when it finished. - */ - public synchronized void notifyFinished() { - if (errCode > 0) return; + synchronized void notifyError(int code, Exception err) { + Log.e(TAG, "notifyError() code = " + code, err); + + errCode = code; + errObject = err; + + pause(); + + notify(DownloadManagerService.MESSAGE_ERROR); + } + + synchronized void notifyFinished() { + if (errCode > ERROR_NOTHING) return; finishCount++; - if (finishCount == threadCount) { - onFinish(); + if (finishCount == currentThreadCount) { + if (errCode > ERROR_NOTHING) return; + + if (DEBUG) { + Log.d(TAG, "onFinish" + (current + 1) + "/" + urls.length); + } + + if ((current + 1) < urls.length) { + // prepare next sub-mission + long current_offset = offsets[current++]; + offsets[current] = current_offset + length; + initializer(); + return; + } + + current++; + unknownLength = false; + + if (!doPostprocessing()) return; + + running = false; + deleteThisFromFile(); + + notify(DownloadManagerService.MESSAGE_FINISHED); } } - /** - * Called when all parts are downloaded - */ - private void onFinish() { - if (errCode > 0) return; - + private void notifyPostProcessing(boolean processing) { if (DEBUG) { - Log.d(TAG, "onFinish"); + Log.d(TAG, (processing ? "enter" : "exit") + " postprocessing on " + location + File.separator + name); } - running = false; - finished = true; - - deleteThisFromFile(); - - for (WeakReference ref : mListeners) { - final MissionListener listener = ref.get(); - if (listener != null) { - MissionListener.handlerStore.get(listener).post(new Runnable() { - @Override - public void run() { - listener.onFinish(DownloadMission.this); - } - }); + synchronized (blockState) { + if (!processing) { + postprocessingName = null; + postprocessingArgs = null; } - } - } - public synchronized void notifyError(int err) { - errCode = err; - - writeThisToFile(); - - for (WeakReference ref : mListeners) { - final MissionListener listener = ref.get(); - MissionListener.handlerStore.get(listener).post(new Runnable() { - @Override - public void run() { - listener.onError(DownloadMission.this, errCode); - } - }); - } - } - - public synchronized void addListener(MissionListener listener) { - Handler handler = new Handler(Looper.getMainLooper()); - MissionListener.handlerStore.put(listener, handler); - mListeners.add(new WeakReference<>(listener)); - } - - public synchronized void removeListener(MissionListener listener) { - for (Iterator> iterator = mListeners.iterator(); - iterator.hasNext(); ) { - WeakReference weakRef = iterator.next(); - if (listener != null && listener == weakRef.get()) { - iterator.remove(); - } + // don't return without fully write the current state + postprocessingRunning = processing; + Utility.writeToFile(metadata, DownloadMission.this); } } @@ -249,92 +403,257 @@ public class DownloadMission implements Serializable { * Start downloading with multiple threads. */ public void start() { - if (!running && !finished) { - running = true; + if (running || current >= urls.length) return; - if (!fallback) { - for (int i = 0; i < threadCount; i++) { - if (threadPositions.size() <= i && !recovered) { - threadPositions.add((long) i); - } - new Thread(new DownloadRunnable(this, i)).start(); - } - } else { - // In fallback mode, resuming is not supported. - threadCount = 1; + // ensure that the previous state is completely paused. + joinForThread(init); + for (Thread thread : threads) joinForThread(thread); + + enqueued = false; + running = true; + errCode = ERROR_NOTHING; + + if (blocks < 0) { + initializer(); + return; + } + + init = null; + + if (threads.length < 1) { + threads = new Thread[currentThreadCount]; + } + + if (fallback) { + if (unknownLength) { done = 0; - blocks = 0; - new Thread(new DownloadRunnableFallback(this)).start(); + length = 0; + } + + threads[0] = runAsync(1, new DownloadRunnableFallback(this)); + } else { + for (int i = 0; i < currentThreadCount; i++) { + threads[i] = runAsync(i + 1, new DownloadRunnable(this, i)); } } } - public void pause() { - if (running) { - running = false; - recovered = true; + /** + * Pause the mission, does not affect the blocks that are being downloaded. + */ + public synchronized void pause() { + if (!running) return; - // TODO: Notify & Write state to info file - // if (err) + running = false; + recovered = true; + enqueued = false; + + if (postprocessingRunning) { + if (DEBUG) { + Log.w(TAG, "pause during post-processing is not applicable."); + } + return; } + + if (init != null && init.isAlive()) { + init.interrupt(); + synchronized (blockState) { + resetState(); + } + return; + } + + if (DEBUG && blocks == 0) { + Log.w(TAG, "pausing a download that can not be resumed (range requests not allowed by the server)."); + } + + if (threads == null || Thread.currentThread().isInterrupted()) { + writeThisToFile(); + return; + } + + // wait for all threads are suspended before save the state + runAsync(-1, () -> { + try { + for (Thread thread : threads) { + if (thread.isAlive()) { + thread.interrupt(); + thread.join(5000); + } + } + } catch (Exception e) { + // nothing to do + } finally { + writeThisToFile(); + } + }); } /** * Removes the file and the meta file */ - public void delete() { - deleteThisFromFile(); - new File(location, name).delete(); + @Override + public boolean delete() { + deleted = true; + boolean res = deleteThisFromFile(); + if (!super.delete()) res = false; + return res; + } + + void resetState() { + done = 0; + blocks = -1; + errCode = ERROR_NOTHING; + fallback = false; + unknownLength = false; + finishCount = 0; + threadBlockPositions.clear(); + threadBytePositions.clear(); + blockState.clear(); + threads = new Thread[0]; + + Utility.writeToFile(metadata, DownloadMission.this); + } + + private void initializer() { + init = runAsync(DownloadInitializer.mId, new DownloadInitializer(this)); + } /** * Write this {@link DownloadMission} to the meta file asynchronously * if no thread is already running. */ - public void writeThisToFile() { - if (!mWritingToFile) { - mWritingToFile = true; - new Thread() { - @Override - public void run() { - doWriteThisToFile(); - mWritingToFile = false; - } - }.start(); - } - } - - /** - * Write this {@link DownloadMission} to the meta file. - */ - private void doWriteThisToFile() { + private void writeThisToFile() { synchronized (blockState) { - Utility.writeToFile(getMetaFilename(), this); + if (deleted) return; + Utility.writeToFile(metadata, DownloadMission.this); + } + mWritingToFile = false; + } + + public boolean isFinished() { + return current >= urls.length && postprocessingName == null; + } + + public long getLength() { + long calculated; + if (postprocessingRunning) { + calculated = length; + } else { + calculated = offsets[current < offsets.length ? current : (offsets.length - 1)] + length; + } + + calculated -= offsets[0];// don't count reserved space + + return calculated > nearLength ? calculated : nearLength; + } + + private boolean doPostprocessing() { + if (postprocessingName == null) return true; + + try { + notifyPostProcessing(true); + notifyProgress(0); + + Thread.currentThread().setName("[" + TAG + "] post-processing = " + postprocessingName + " filename = " + name); + + Postprocessing algorithm = Postprocessing.getAlgorithm(postprocessingName, this); + algorithm.run(); + } catch (Exception err) { + StringBuilder args = new StringBuilder(" "); + if (postprocessingArgs != null) { + for (String arg : postprocessingArgs) { + args.append(", "); + args.append(arg); + } + args.delete(0, 1); + } + Log.e(TAG, String.format("Post-processing failed. algorithm = %s args = [%s]", postprocessingName, args), err); + + notifyError(ERROR_POSTPROCESSING_FAILED, err); + return false; + } finally { + notifyPostProcessing(false); + } + + if (errCode != ERROR_NOTHING) notify(DownloadManagerService.MESSAGE_ERROR); + + return errCode == ERROR_NOTHING; + } + + private boolean deleteThisFromFile() { + synchronized (blockState) { + return metadata.delete(); } } - private void readObject(ObjectInputStream inputStream) - throws java.io.IOException, ClassNotFoundException - { - inputStream.defaultReadObject(); - mListeners = new ArrayList<>(); - } - - private void deleteThisFromFile() { - new File(getMetaFilename()).delete(); + /** + * run a new thread + * + * @param id id of new thread (used for debugging only) + * @param who the Runnable whose {@code run} method is invoked. + */ + private void runAsync(int id, Runnable who) { + runAsync(id, new Thread(who)); } /** - * Get the path of the meta file + * run a new thread * - * @return the path to the meta file + * @param id id of new thread (used for debugging only) + * @param who the Thread whose {@code run} method is invoked when this thread is started + * @return the passed thread */ - private String getMetaFilename() { - return location + "/" + name + ".giga"; + private Thread runAsync(int id, Thread who) { + // known thread ids: + // -2: state saving by notifyProgress() method + // -1: wait for saving the state by pause() method + // 0: initializer + // >=1: any download thread + + if (DEBUG) { + who.setName(String.format("%s[%s] %s", TAG, id, name)); + } + + who.start(); + + return who; } - public File getDownloadedFile() { - return new File(location, name); + private void joinForThread(Thread thread) { + if (thread == null || !thread.isAlive()) return; + if (thread == Thread.currentThread()) return; + + if (DEBUG) { + Log.w(TAG, "a thread is !still alive!: " + thread.getName()); + } + + // still alive, this should not happen. + // Possible reasons: + // slow device + // the user is spamming start/pause buttons + // start() method called quickly after pause() + + try { + thread.join(10000); + } catch (InterruptedException e) { + Log.d(TAG, "timeout on join : " + thread.getName()); + throw new RuntimeException("A thread is still running:\n" + thread.getName()); + } } + + static class HttpError extends Exception { + int statusCode; + + HttpError(int statusCode) { + this.statusCode = statusCode; + } + + @Override + public String getMessage() { + return "HTTP " + String.valueOf(statusCode); + } + } } diff --git a/app/src/main/java/us/shandian/giga/get/DownloadRunnable.java b/app/src/main/java/us/shandian/giga/get/DownloadRunnable.java index 6ad8626c3..244fbd47a 100644 --- a/app/src/main/java/us/shandian/giga/get/DownloadRunnable.java +++ b/app/src/main/java/us/shandian/giga/get/DownloadRunnable.java @@ -2,9 +2,11 @@ package us.shandian.giga.get; import android.util.Log; +import java.io.FileNotFoundException; +import java.io.InputStream; import java.io.RandomAccessFile; import java.net.HttpURLConnection; -import java.net.URL; +import java.nio.channels.ClosedByInterruptException; import static org.schabi.newpipe.BuildConfig.DEBUG; @@ -12,142 +14,166 @@ import static org.schabi.newpipe.BuildConfig.DEBUG; * Runnable to download blocks of a file until the file is completely downloaded, * an error occurs or the process is stopped. */ -public class DownloadRunnable implements Runnable { +public class DownloadRunnable extends Thread { private static final String TAG = DownloadRunnable.class.getSimpleName(); private final DownloadMission mMission; private final int mId; - public DownloadRunnable(DownloadMission mission, int id) { + private HttpURLConnection mConn; + + DownloadRunnable(DownloadMission mission, int id) { if (mission == null) throw new NullPointerException("mission is null"); mMission = mission; mId = id; + mConn = null; } @Override public void run() { boolean retry = mMission.recovered; - long position = mMission.getPosition(mId); + long blockPosition = mMission.getBlockPosition(mId); + int retryCount = 0; if (DEBUG) { - Log.d(TAG, mId + ":default pos " + position); + Log.d(TAG, mId + ":default pos " + blockPosition); Log.d(TAG, mId + ":recovered: " + mMission.recovered); } - while (mMission.errCode == -1 && mMission.running && position < mMission.blocks) { + RandomAccessFile f; + InputStream is = null; - if (Thread.currentThread().isInterrupted()) { - mMission.pause(); - return; - } + try { + f = new RandomAccessFile(mMission.getDownloadedFile(), "rw"); + } catch (FileNotFoundException e) { + mMission.notifyError(e);// this never should happen + return; + } + + while (mMission.running && mMission.errCode == DownloadMission.ERROR_NOTHING && blockPosition < mMission.blocks) { if (DEBUG && retry) { - Log.d(TAG, mId + ":retry is true. Resuming at " + position); + Log.d(TAG, mId + ":retry is true. Resuming at " + blockPosition); } // Wait for an unblocked position - while (!retry && position < mMission.blocks && mMission.isBlockPreserved(position)) { + while (!retry && blockPosition < mMission.blocks && mMission.isBlockPreserved(blockPosition)) { if (DEBUG) { - Log.d(TAG, mId + ":position " + position + " preserved, passing"); + Log.d(TAG, mId + ":position " + blockPosition + " preserved, passing"); } - position++; + blockPosition++; } retry = false; - if (position >= mMission.blocks) { + if (blockPosition >= mMission.blocks) { break; } if (DEBUG) { - Log.d(TAG, mId + ":preserving position " + position); + Log.d(TAG, mId + ":preserving position " + blockPosition); } - mMission.preserveBlock(position); - mMission.setPosition(mId, position); + mMission.preserveBlock(blockPosition); + mMission.setBlockPosition(mId, blockPosition); - long start = position * DownloadManager.BLOCK_SIZE; - long end = start + DownloadManager.BLOCK_SIZE - 1; + long start = blockPosition * DownloadMission.BLOCK_SIZE; + long end = start + DownloadMission.BLOCK_SIZE - 1; + long offset = mMission.getThreadBytePosition(mId); + + start += offset; if (end >= mMission.length) { end = mMission.length - 1; } - HttpURLConnection conn = null; - - int total = 0; + long total = 0; try { - URL url = new URL(mMission.url); - conn = (HttpURLConnection) url.openConnection(); - conn.setRequestProperty("Range", "bytes=" + start + "-" + end); + mConn = mMission.openConnection(mId, start, end); + mMission.establishConnection(mId, mConn); - if (DEBUG) { - Log.d(TAG, mId + ":" + conn.getRequestProperty("Range")); - Log.d(TAG, mId + ":Content-Length=" + conn.getContentLength() + " Code:" + conn.getResponseCode()); + // check if the download can be resumed + if (mConn.getResponseCode() == 416 && offset > 0) { + retryCount--; + throw new DownloadMission.HttpError(416); } - // A server may be ignoring the range request - if (conn.getResponseCode() != 206) { - mMission.errCode = DownloadMission.ERROR_SERVER_UNSUPPORTED; - notifyError(); + // The server may be ignoring the range request + if (mConn.getResponseCode() != 206) { + mMission.notifyError(new DownloadMission.HttpError(mConn.getResponseCode())); if (DEBUG) { - Log.e(TAG, mId + ":Unsupported " + conn.getResponseCode()); + Log.e(TAG, mId + ":Unsupported " + mConn.getResponseCode()); } break; } - RandomAccessFile f = new RandomAccessFile(mMission.location + "/" + mMission.name, "rw"); - f.seek(start); - java.io.InputStream ipt = conn.getInputStream(); - byte[] buf = new byte[64*1024]; + f.seek(mMission.offsets[mMission.current] + start); - while (start < end && mMission.running) { - int len = ipt.read(buf, 0, buf.length); + is = mConn.getInputStream(); - if (len == -1) { - break; - } else { - start += len; - total += len; - f.write(buf, 0, len); - notifyProgress(len); - } + byte[] buf = new byte[DownloadMission.BUFFER_SIZE]; + int len; + + while (start < end && mMission.running && (len = is.read(buf, 0, buf.length)) != -1) { + f.write(buf, 0, len); + start += len; + total += len; + mMission.notifyProgress(len); } if (DEBUG && mMission.running) { - Log.d(TAG, mId + ":position " + position + " finished, total length " + total); + Log.d(TAG, mId + ":position " + blockPosition + " finished, " + total + " bytes downloaded"); } - f.close(); - ipt.close(); + if (mMission.running) + mMission.setThreadBytePosition(mId, 0L);// clear byte position for next block + else + mMission.setThreadBytePosition(mId, total);// download paused, save progress for this block - // TODO We should save progress for each thread } catch (Exception e) { - // TODO Retry count limit & notify error - retry = true; + mMission.setThreadBytePosition(mId, total); - notifyProgress(-total); + if (!mMission.running || e instanceof ClosedByInterruptException) break; + + if (retryCount++ >= mMission.maxRetry) { + mMission.notifyError(e); + break; + } if (DEBUG) { - Log.d(TAG, mId + ":position " + position + " retrying", e); + Log.d(TAG, mId + ":position " + blockPosition + " retrying due exception", e); } + + retry = true; } } + try { + if (is != null) is.close(); + } catch (Exception err) { + // nothing to do + } + + try { + f.close(); + } catch (Exception err) { + // ¿ejected media storage? ¿file deleted? ¿storage ran out of space? + } + if (DEBUG) { - Log.d(TAG, "thread " + mId + " exited main loop"); + Log.d(TAG, "thread " + mId + " exited from main download loop"); } - if (mMission.errCode == -1 && mMission.running) { + if (mMission.errCode == DownloadMission.ERROR_NOTHING && mMission.running) { if (DEBUG) { Log.d(TAG, "no error has happened, notifying"); } - notifyFinished(); + mMission.notifyFinished(); } if (DEBUG && !mMission.running) { @@ -155,22 +181,15 @@ public class DownloadRunnable implements Runnable { } } - private void notifyProgress(final long len) { - synchronized (mMission) { - mMission.notifyProgress(len); + @Override + public void interrupt() { + super.interrupt(); + + try { + if (mConn != null) mConn.disconnect(); + } catch (Exception e) { + // nothing to do } } - private void notifyError() { - synchronized (mMission) { - mMission.notifyError(DownloadMission.ERROR_SERVER_UNSUPPORTED); - mMission.pause(); - } - } - - private void notifyFinished() { - synchronized (mMission) { - mMission.notifyFinished(); - } - } } diff --git a/app/src/main/java/us/shandian/giga/get/DownloadRunnableFallback.java b/app/src/main/java/us/shandian/giga/get/DownloadRunnableFallback.java index f24139910..4bcaeaf85 100644 --- a/app/src/main/java/us/shandian/giga/get/DownloadRunnableFallback.java +++ b/app/src/main/java/us/shandian/giga/get/DownloadRunnableFallback.java @@ -1,74 +1,139 @@ package us.shandian.giga.get; -import java.io.BufferedInputStream; +import android.annotation.SuppressLint; +import android.support.annotation.NonNull; +import android.util.Log; + +import java.io.IOException; +import java.io.InputStream; import java.io.RandomAccessFile; import java.net.HttpURLConnection; -import java.net.URL; +import java.nio.channels.ClosedByInterruptException; + + +import us.shandian.giga.util.Utility; + +import static org.schabi.newpipe.BuildConfig.DEBUG; + +/** + * Single-threaded fallback mode + */ +public class DownloadRunnableFallback extends Thread { + private static final String TAG = "DownloadRunnableFallback"; -// Single-threaded fallback mode -public class DownloadRunnableFallback implements Runnable { private final DownloadMission mMission; - //private int mId; + private final int mId = 1; - public DownloadRunnableFallback(DownloadMission mission) { - if (mission == null) throw new NullPointerException("mission is null"); - //mId = id; + private int mRetryCount = 0; + private InputStream mIs; + private RandomAccessFile mF; + private HttpURLConnection mConn; + + DownloadRunnableFallback(@NonNull DownloadMission mission) { mMission = mission; + mIs = null; + mF = null; + mConn = null; + } + + private void dispose() { + try { + if (mIs != null) mIs.close(); + } catch (IOException e) { + // nothing to do + } + + try { + if (mF != null) mF.close(); + } catch (IOException e) { + // ¿ejected media storage? ¿file deleted? ¿storage ran out of space? + } } @Override + @SuppressLint("LongLogTag") public void run() { - try { - URL url = new URL(mMission.url); - HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + boolean done; - if (conn.getResponseCode() != 200 && conn.getResponseCode() != 206) { - notifyError(DownloadMission.ERROR_SERVER_UNSUPPORTED); - } else { - RandomAccessFile f = new RandomAccessFile(mMission.location + "/" + mMission.name, "rw"); - f.seek(0); - BufferedInputStream ipt = new BufferedInputStream(conn.getInputStream()); - byte[] buf = new byte[512]; - int len = 0; + long start = 0; - while ((len = ipt.read(buf, 0, 512)) != -1 && mMission.running) { - f.write(buf, 0, len); - notifyProgress(len); - - if (Thread.interrupted()) { - break; - } - - } - - f.close(); - ipt.close(); + if (!mMission.unknownLength) { + start = mMission.getThreadBytePosition(0); + if (DEBUG && start > 0) { + Log.i(TAG, "Resuming a single-thread download at " + start); } + } + + try { + long rangeStart = (mMission.unknownLength || start < 1) ? -1 : start; + + mConn = mMission.openConnection(mId, rangeStart, -1); + mMission.establishConnection(mId, mConn); + + // check if the download can be resumed + if (mConn.getResponseCode() == 416 && start > 0) { + start = 0; + mRetryCount--; + throw new DownloadMission.HttpError(416); + } + + // secondary check for the file length + if (!mMission.unknownLength) + mMission.unknownLength = Utility.getContentLength(mConn) == -1; + + mF = new RandomAccessFile(mMission.getDownloadedFile(), "rw"); + mF.seek(mMission.offsets[mMission.current] + start); + + mIs = mConn.getInputStream(); + + byte[] buf = new byte[64 * 1024]; + int len = 0; + + while (mMission.running && (len = mIs.read(buf, 0, buf.length)) != -1) { + mF.write(buf, 0, len); + start += len; + mMission.notifyProgress(len); + } + + // if thread goes interrupted check if the last part mIs written. This avoid re-download the whole file + done = len == -1; } catch (Exception e) { - notifyError(DownloadMission.ERROR_UNKNOWN); + dispose(); + + // save position + mMission.setThreadBytePosition(0, start); + + if (!mMission.running || e instanceof ClosedByInterruptException) return; + + if (mRetryCount++ >= mMission.maxRetry) { + mMission.notifyError(e); + return; + } + + run();// try again + return; } - if (mMission.errCode == -1 && mMission.running) { - notifyFinished(); - } - } + dispose(); - private void notifyProgress(final long len) { - synchronized (mMission) { - mMission.notifyProgress(len); - } - } - - private void notifyError(final int err) { - synchronized (mMission) { - mMission.notifyError(err); - mMission.pause(); - } - } - - private void notifyFinished() { - synchronized (mMission) { + if (done) { mMission.notifyFinished(); + } else { + mMission.setThreadBytePosition(0, start); + } + } + + @Override + public void interrupt() { + super.interrupt(); + + if (mConn != null) { + try { + mConn.disconnect(); + } catch (Exception e) { + // nothing to do + } + } } } diff --git a/app/src/main/java/us/shandian/giga/get/FinishedMission.java b/app/src/main/java/us/shandian/giga/get/FinishedMission.java new file mode 100644 index 000000000..b7d6908a5 --- /dev/null +++ b/app/src/main/java/us/shandian/giga/get/FinishedMission.java @@ -0,0 +1,16 @@ +package us.shandian.giga.get; + +public class FinishedMission extends Mission { + + public FinishedMission() { + } + + public FinishedMission(DownloadMission mission) { + source = mission.source; + length = mission.length;// ¿or mission.done? + timestamp = mission.timestamp; + name = mission.name; + location = mission.location; + kind = mission.kind; + } +} diff --git a/app/src/main/java/us/shandian/giga/get/Mission.java b/app/src/main/java/us/shandian/giga/get/Mission.java new file mode 100644 index 000000000..ec2ddaa26 --- /dev/null +++ b/app/src/main/java/us/shandian/giga/get/Mission.java @@ -0,0 +1,66 @@ +package us.shandian.giga.get; + +import java.io.File; +import java.io.Serializable; +import java.text.SimpleDateFormat; +import java.util.Calendar; + +public abstract class Mission implements Serializable { + private static final long serialVersionUID = 0L;// last bump: 5 october 2018 + + /** + * Source url of the resource + */ + public String source; + + /** + * Length of the current resource + */ + public long length; + + /** + * creation timestamp (and maybe unique identifier) + */ + public long timestamp; + + /** + * The filename + */ + public String name; + + /** + * The directory to store the download + */ + public String location; + + /** + * pre-defined content type + */ + public char kind; + + /** + * get the target file on the storage + * + * @return File object + */ + public File getDownloadedFile() { + return new File(location, name); + } + + public boolean delete() { + deleted = true; + return getDownloadedFile().delete(); + } + + /** + * Indicate if this mission is deleted whatever is stored + */ + public transient boolean deleted = false; + + @Override + public String toString() { + Calendar calendar = Calendar.getInstance(); + calendar.setTimeInMillis(timestamp); + return "[" + calendar.getTime().toString() + "] " + location + File.separator + name; + } +} diff --git a/app/src/main/java/us/shandian/giga/get/sqlite/DownloadDataSource.java b/app/src/main/java/us/shandian/giga/get/sqlite/DownloadDataSource.java new file mode 100644 index 000000000..4b4d5d733 --- /dev/null +++ b/app/src/main/java/us/shandian/giga/get/sqlite/DownloadDataSource.java @@ -0,0 +1,73 @@ +package us.shandian.giga.get.sqlite; + +import android.content.ContentValues; +import android.content.Context; +import android.database.Cursor; +import android.database.sqlite.SQLiteDatabase; +import android.util.Log; + +import java.util.ArrayList; +import java.util.List; + +import us.shandian.giga.get.DownloadMission; +import us.shandian.giga.get.FinishedMission; +import us.shandian.giga.get.Mission; + +import static us.shandian.giga.get.sqlite.DownloadMissionHelper.KEY_LOCATION; +import static us.shandian.giga.get.sqlite.DownloadMissionHelper.KEY_NAME; +import static us.shandian.giga.get.sqlite.DownloadMissionHelper.MISSIONS_TABLE_NAME; + +public class DownloadDataSource { + + private static final String TAG = "DownloadDataSource"; + private final DownloadMissionHelper downloadMissionHelper; + + public DownloadDataSource(Context context) { + downloadMissionHelper = new DownloadMissionHelper(context); + } + + public ArrayList loadFinishedMissions() { + SQLiteDatabase database = downloadMissionHelper.getReadableDatabase(); + Cursor cursor = database.query(MISSIONS_TABLE_NAME, null, null, + null, null, null, DownloadMissionHelper.KEY_TIMESTAMP); + + int count = cursor.getCount(); + if (count == 0) return new ArrayList<>(1); + + ArrayList result = new ArrayList<>(count); + while (cursor.moveToNext()) { + result.add(DownloadMissionHelper.getMissionFromCursor(cursor)); + } + + return result; + } + + public void addMission(DownloadMission downloadMission) { + if (downloadMission == null) throw new NullPointerException("downloadMission is null"); + SQLiteDatabase database = downloadMissionHelper.getWritableDatabase(); + ContentValues values = DownloadMissionHelper.getValuesOfMission(downloadMission); + database.insert(MISSIONS_TABLE_NAME, null, values); + } + + public void deleteMission(Mission downloadMission) { + if (downloadMission == null) throw new NullPointerException("downloadMission is null"); + SQLiteDatabase database = downloadMissionHelper.getWritableDatabase(); + database.delete(MISSIONS_TABLE_NAME, + KEY_LOCATION + " = ? AND " + + KEY_NAME + " = ?", + new String[]{downloadMission.location, downloadMission.name}); + } + + public void updateMission(DownloadMission downloadMission) { + if (downloadMission == null) throw new NullPointerException("downloadMission is null"); + SQLiteDatabase database = downloadMissionHelper.getWritableDatabase(); + ContentValues values = DownloadMissionHelper.getValuesOfMission(downloadMission); + String whereClause = KEY_LOCATION + " = ? AND " + + KEY_NAME + " = ?"; + int rowsAffected = database.update(MISSIONS_TABLE_NAME, values, + whereClause, new String[]{downloadMission.location, downloadMission.name}); + if (rowsAffected != 1) { + Log.e(TAG, "Expected 1 row to be affected by update but got " + rowsAffected); + } + } +} diff --git a/app/src/main/java/us/shandian/giga/get/sqlite/DownloadMissionSQLiteHelper.java b/app/src/main/java/us/shandian/giga/get/sqlite/DownloadMissionHelper.java similarity index 63% rename from app/src/main/java/us/shandian/giga/get/sqlite/DownloadMissionSQLiteHelper.java rename to app/src/main/java/us/shandian/giga/get/sqlite/DownloadMissionHelper.java index d5a83551b..6dadc98c8 100644 --- a/app/src/main/java/us/shandian/giga/get/sqlite/DownloadMissionSQLiteHelper.java +++ b/app/src/main/java/us/shandian/giga/get/sqlite/DownloadMissionHelper.java @@ -7,19 +7,19 @@ import android.database.sqlite.SQLiteDatabase; import android.database.sqlite.SQLiteOpenHelper; import us.shandian.giga.get.DownloadMission; +import us.shandian.giga.get.FinishedMission; /** - * SqliteHelper to store {@link us.shandian.giga.get.DownloadMission} + * SQLiteHelper to store finished {@link us.shandian.giga.get.DownloadMission}'s */ -public class DownloadMissionSQLiteHelper extends SQLiteOpenHelper { - - +public class DownloadMissionHelper extends SQLiteOpenHelper { private final String TAG = "DownloadMissionHelper"; // TODO: use NewPipeSQLiteHelper ('s constants) when playlist branch is merged (?) private static final String DATABASE_NAME = "downloads.db"; - private static final int DATABASE_VERSION = 2; + private static final int DATABASE_VERSION = 3; + /** * The table name of download missions */ @@ -30,9 +30,9 @@ public class DownloadMissionSQLiteHelper extends SQLiteOpenHelper { */ static final String KEY_LOCATION = "location"; /** - * The key to the url of a mission + * The key to the urls of a mission */ - static final String KEY_URL = "url"; + static final String KEY_SOURCE_URL = "url"; /** * The key to the name of a mission */ @@ -45,6 +45,8 @@ public class DownloadMissionSQLiteHelper extends SQLiteOpenHelper { static final String KEY_TIMESTAMP = "timestamp"; + static final String KEY_KIND = "kind"; + /** * The statement to create the table */ @@ -52,16 +54,28 @@ public class DownloadMissionSQLiteHelper extends SQLiteOpenHelper { "CREATE TABLE " + MISSIONS_TABLE_NAME + " (" + KEY_LOCATION + " TEXT NOT NULL, " + KEY_NAME + " TEXT NOT NULL, " + - KEY_URL + " TEXT NOT NULL, " + + KEY_SOURCE_URL + " TEXT NOT NULL, " + KEY_DONE + " INTEGER NOT NULL, " + KEY_TIMESTAMP + " INTEGER NOT NULL, " + + KEY_KIND + " TEXT NOT NULL, " + " UNIQUE(" + KEY_LOCATION + ", " + KEY_NAME + "));"; - - DownloadMissionSQLiteHelper(Context context) { + public DownloadMissionHelper(Context context) { super(context, DATABASE_NAME, null, DATABASE_VERSION); } + @Override + public void onCreate(SQLiteDatabase db) { + db.execSQL(MISSIONS_CREATE_TABLE); + } + + @Override + public void onUpgrade(SQLiteDatabase db, int oldVersion, int newVersion) { + if (oldVersion == 2) { + db.execSQL("ALTER TABLE " + MISSIONS_TABLE_NAME + " ADD COLUMN " + KEY_KIND + " TEXT;"); + } + } + /** * Returns all values of the download mission as ContentValues. * @@ -70,34 +84,29 @@ public class DownloadMissionSQLiteHelper extends SQLiteOpenHelper { */ public static ContentValues getValuesOfMission(DownloadMission downloadMission) { ContentValues values = new ContentValues(); - values.put(KEY_URL, downloadMission.url); + values.put(KEY_SOURCE_URL, downloadMission.source); values.put(KEY_LOCATION, downloadMission.location); values.put(KEY_NAME, downloadMission.name); values.put(KEY_DONE, downloadMission.done); values.put(KEY_TIMESTAMP, downloadMission.timestamp); + values.put(KEY_KIND, String.valueOf(downloadMission.kind)); return values; } - @Override - public void onCreate(SQLiteDatabase db) { - db.execSQL(MISSIONS_CREATE_TABLE); - } - - @Override - public void onUpgrade(SQLiteDatabase db, int oldVersion, int newVersion) { - // Currently nothing to do - } - - public static DownloadMission getMissionFromCursor(Cursor cursor) { + public static FinishedMission getMissionFromCursor(Cursor cursor) { if (cursor == null) throw new NullPointerException("cursor is null"); - int pos; - String name = cursor.getString(cursor.getColumnIndexOrThrow(KEY_NAME)); - String location = cursor.getString(cursor.getColumnIndexOrThrow(KEY_LOCATION)); - String url = cursor.getString(cursor.getColumnIndexOrThrow(KEY_URL)); - DownloadMission mission = new DownloadMission(name, url, location); - mission.done = cursor.getLong(cursor.getColumnIndexOrThrow(KEY_DONE)); + + String kind = cursor.getString(cursor.getColumnIndex(KEY_KIND)); + if (kind == null || kind.isEmpty()) kind = "?"; + + FinishedMission mission = new FinishedMission(); + mission.name = cursor.getString(cursor.getColumnIndexOrThrow(KEY_NAME)); + mission.location = cursor.getString(cursor.getColumnIndexOrThrow(KEY_LOCATION)); + mission.source = cursor.getString(cursor.getColumnIndexOrThrow(KEY_SOURCE_URL));; + mission.length = cursor.getLong(cursor.getColumnIndexOrThrow(KEY_DONE)); mission.timestamp = cursor.getLong(cursor.getColumnIndexOrThrow(KEY_TIMESTAMP)); - mission.finished = true; + mission.kind = kind.charAt(0); + return mission; } } diff --git a/app/src/main/java/us/shandian/giga/get/sqlite/SQLiteDownloadDataSource.java b/app/src/main/java/us/shandian/giga/get/sqlite/SQLiteDownloadDataSource.java deleted file mode 100644 index e7b4caeb8..000000000 --- a/app/src/main/java/us/shandian/giga/get/sqlite/SQLiteDownloadDataSource.java +++ /dev/null @@ -1,79 +0,0 @@ -package us.shandian.giga.get.sqlite; - -import android.content.ContentValues; -import android.content.Context; -import android.database.Cursor; -import android.database.sqlite.SQLiteDatabase; -import android.util.Log; - -import java.util.ArrayList; -import java.util.List; - -import us.shandian.giga.get.DownloadDataSource; -import us.shandian.giga.get.DownloadMission; - -import static us.shandian.giga.get.sqlite.DownloadMissionSQLiteHelper.KEY_LOCATION; -import static us.shandian.giga.get.sqlite.DownloadMissionSQLiteHelper.KEY_NAME; -import static us.shandian.giga.get.sqlite.DownloadMissionSQLiteHelper.MISSIONS_TABLE_NAME; - - -/** - * Non-thread-safe implementation of {@link DownloadDataSource} - */ -public class SQLiteDownloadDataSource implements DownloadDataSource { - - private static final String TAG = "DownloadDataSourceImpl"; - private final DownloadMissionSQLiteHelper downloadMissionSQLiteHelper; - - public SQLiteDownloadDataSource(Context context) { - downloadMissionSQLiteHelper = new DownloadMissionSQLiteHelper(context); - } - - @Override - public List loadMissions() { - ArrayList result; - SQLiteDatabase database = downloadMissionSQLiteHelper.getReadableDatabase(); - Cursor cursor = database.query(MISSIONS_TABLE_NAME, null, null, - null, null, null, DownloadMissionSQLiteHelper.KEY_TIMESTAMP); - - int count = cursor.getCount(); - if (count == 0) return new ArrayList<>(); - result = new ArrayList<>(count); - while (cursor.moveToNext()) { - result.add(DownloadMissionSQLiteHelper.getMissionFromCursor(cursor)); - } - return result; - } - - @Override - public void addMission(DownloadMission downloadMission) { - if (downloadMission == null) throw new NullPointerException("downloadMission is null"); - SQLiteDatabase database = downloadMissionSQLiteHelper.getWritableDatabase(); - ContentValues values = DownloadMissionSQLiteHelper.getValuesOfMission(downloadMission); - database.insert(MISSIONS_TABLE_NAME, null, values); - } - - @Override - public void updateMission(DownloadMission downloadMission) { - if (downloadMission == null) throw new NullPointerException("downloadMission is null"); - SQLiteDatabase database = downloadMissionSQLiteHelper.getWritableDatabase(); - ContentValues values = DownloadMissionSQLiteHelper.getValuesOfMission(downloadMission); - String whereClause = KEY_LOCATION + " = ? AND " + - KEY_NAME + " = ?"; - int rowsAffected = database.update(MISSIONS_TABLE_NAME, values, - whereClause, new String[]{downloadMission.location, downloadMission.name}); - if (rowsAffected != 1) { - Log.e(TAG, "Expected 1 row to be affected by update but got " + rowsAffected); - } - } - - @Override - public void deleteMission(DownloadMission downloadMission) { - if (downloadMission == null) throw new NullPointerException("downloadMission is null"); - SQLiteDatabase database = downloadMissionSQLiteHelper.getWritableDatabase(); - database.delete(MISSIONS_TABLE_NAME, - KEY_LOCATION + " = ? AND " + - KEY_NAME + " = ?", - new String[]{downloadMission.location, downloadMission.name}); - } -} diff --git a/app/src/main/java/us/shandian/giga/postprocessing/Mp4DashMuxer.java b/app/src/main/java/us/shandian/giga/postprocessing/Mp4DashMuxer.java new file mode 100644 index 000000000..b303b66cd --- /dev/null +++ b/app/src/main/java/us/shandian/giga/postprocessing/Mp4DashMuxer.java @@ -0,0 +1,31 @@ +package us.shandian.giga.postprocessing; + +import org.schabi.newpipe.streams.Mp4DashWriter; +import org.schabi.newpipe.streams.io.SharpStream; + +import java.io.IOException; + +import us.shandian.giga.get.DownloadMission; + +/** + * @author kapodamy + */ +class Mp4DashMuxer extends Postprocessing { + + Mp4DashMuxer(DownloadMission mission) { + super(mission); + recommendedReserve = 15360 * 1024;// 15 MiB + worksOnSameFile = true; + } + + @Override + int process(SharpStream out, SharpStream... sources) throws IOException { + Mp4DashWriter muxer = new Mp4DashWriter(sources); + muxer.parseSources(); + muxer.selectTracks(0, 0); + muxer.build(out); + + return OK_RESULT; + } + +} diff --git a/app/src/main/java/us/shandian/giga/postprocessing/Postprocessing.java b/app/src/main/java/us/shandian/giga/postprocessing/Postprocessing.java new file mode 100644 index 000000000..80726f705 --- /dev/null +++ b/app/src/main/java/us/shandian/giga/postprocessing/Postprocessing.java @@ -0,0 +1,151 @@ +package us.shandian.giga.postprocessing; + +import android.os.Message; + +import org.schabi.newpipe.streams.io.SharpStream; + +import java.io.File; +import java.io.IOException; + +import us.shandian.giga.get.DownloadMission; +import us.shandian.giga.postprocessing.io.ChunkFileInputStream; +import us.shandian.giga.postprocessing.io.CircularFile; +import us.shandian.giga.service.DownloadManagerService; + +public abstract class Postprocessing { + + static final byte OK_RESULT = DownloadMission.ERROR_NOTHING; + + public static final String ALGORITHM_TTML_CONVERTER = "ttml"; + public static final String ALGORITHM_MP4_DASH_MUXER = "mp4D"; + public static final String ALGORITHM_WEBM_MUXER = "webm"; + private static final String ALGORITHM_TEST_ALGO = "test"; + + public static Postprocessing getAlgorithm(String algorithmName, DownloadMission mission) { + if (null == algorithmName) { + throw new NullPointerException("algorithmName"); + } else switch (algorithmName) { + case ALGORITHM_TTML_CONVERTER: + return new TttmlConverter(mission); + case ALGORITHM_MP4_DASH_MUXER: + return new Mp4DashMuxer(mission); + case ALGORITHM_WEBM_MUXER: + return new WebMMuxer(mission); + case ALGORITHM_TEST_ALGO: + return new TestAlgo(mission); + /*case "example-algorithm": + return new ExampleAlgorithm(mission);*/ + default: + throw new RuntimeException("Unimplemented post-processing algorithm: " + algorithmName); + } + } + + /** + * Get a boolean value that indicate if the given algorithm work on the same + * file + */ + public boolean worksOnSameFile; + + /** + * Get the recommended space to reserve for the given algorithm. The amount + * is in bytes + */ + public int recommendedReserve; + + protected DownloadMission mission; + + Postprocessing(DownloadMission mission) { + this.mission = mission; + } + + public void run() throws IOException { + File file = mission.getDownloadedFile(); + CircularFile out = null; + ChunkFileInputStream[] sources = new ChunkFileInputStream[mission.urls.length]; + + try { + int i = 0; + for (; i < sources.length - 1; i++) { + sources[i] = new ChunkFileInputStream(file, mission.offsets[i], mission.offsets[i + 1], "rw"); + } + sources[i] = new ChunkFileInputStream(file, mission.offsets[i], mission.getDownloadedFile().length(), "rw"); + + int[] idx = {0}; + CircularFile.OffsetChecker checker = () -> { + while (idx[0] < sources.length) { + /* + * WARNING: never use rewind() in any chunk after any writing (especially on first chunks) + * or the CircularFile can lead to unexpected results + */ + if (sources[idx[0]].isDisposed() || sources[idx[0]].available() < 1) { + idx[0]++; + continue;// the selected source is not used anymore + } + + return sources[idx[0]].getFilePointer() - 1; + } + + return -1; + }; + + out = new CircularFile(file, 0, this::progressReport, checker); + + mission.done = 0; + mission.length = file.length(); + + int result = process(out, sources); + + if (result == OK_RESULT) { + long finalLength = out.finalizeFile(); + mission.done = finalLength; + mission.length = finalLength; + } else { + mission.errCode = DownloadMission.ERROR_UNKNOWN_EXCEPTION; + mission.errObject = new RuntimeException("post-processing algorithm returned " + result); + } + + if (result != OK_RESULT && worksOnSameFile) { + //noinspection ResultOfMethodCallIgnored + new File(mission.location, mission.name).delete(); + } + } finally { + for (SharpStream source : sources) { + if (source != null && !source.isDisposed()) { + source.dispose(); + } + } + if (out != null) { + out.dispose(); + } + } + } + + /** + * Abstract method to execute the pos-processing algorithm + * + * @param out output stream + * @param sources files to be processed + * @return a error code, 0 means the operation was successful + * @throws IOException if an I/O error occurs. + */ + abstract int process(SharpStream out, SharpStream... sources) throws IOException; + + String getArgumentAt(int index, String defaultValue) { + if (mission.postprocessingArgs == null || index >= mission.postprocessingArgs.length) { + return defaultValue; + } + + return mission.postprocessingArgs[index]; + } + + private void progressReport(long done) { + mission.done = done; + if (mission.length < mission.done) mission.length = mission.done; + + Message m = new Message(); + m.what = DownloadManagerService.MESSAGE_PROGRESS; + m.obj = mission; + + mission.mHandler.sendMessage(m); + } +} diff --git a/app/src/main/java/us/shandian/giga/postprocessing/TestAlgo.java b/app/src/main/java/us/shandian/giga/postprocessing/TestAlgo.java new file mode 100644 index 000000000..66b235d7c --- /dev/null +++ b/app/src/main/java/us/shandian/giga/postprocessing/TestAlgo.java @@ -0,0 +1,54 @@ +package us.shandian.giga.postprocessing; + +import android.util.Log; + +import org.schabi.newpipe.streams.io.SharpStream; + +import java.io.IOException; +import java.util.Random; + +import us.shandian.giga.get.DownloadMission; + +/** + * Algorithm for testing proposes + */ +class TestAlgo extends Postprocessing { + + public TestAlgo(DownloadMission mission) { + super(mission); + + worksOnSameFile = true; + recommendedReserve = 4096 * 1024;// 4 KiB + } + + @Override + int process(SharpStream out, SharpStream... sources) throws IOException { + + int written = 0; + int size = 5 * 1024 * 1024;// 5 MiB + byte[] buffer = new byte[8 * 1024];//8 KiB + mission.length = size; + + Random rnd = new Random(); + + // only write random data + sources[0].dispose(); + + while (written < size) { + rnd.nextBytes(buffer); + + int read = Math.min(buffer.length, size - written); + out.write(buffer, 0, read); + + try { + Thread.sleep((int) (Math.random() * 10)); + } catch (InterruptedException e) { + return -1; + } + + written += read; + } + + return Postprocessing.OK_RESULT; + } +} diff --git a/app/src/main/java/us/shandian/giga/postprocessing/TttmlConverter.java b/app/src/main/java/us/shandian/giga/postprocessing/TttmlConverter.java new file mode 100644 index 000000000..4c9d44548 --- /dev/null +++ b/app/src/main/java/us/shandian/giga/postprocessing/TttmlConverter.java @@ -0,0 +1,76 @@ +package us.shandian.giga.postprocessing; + +import android.util.Log; + +import org.schabi.newpipe.streams.io.SharpStream; +import org.schabi.newpipe.streams.SubtitleConverter; +import org.xml.sax.SAXException; + +import java.io.IOException; +import java.text.ParseException; + +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.xpath.XPathExpressionException; + +import us.shandian.giga.get.DownloadMission; +import us.shandian.giga.postprocessing.io.SharpInputStream; + +/** + * @author kapodamy + */ +class TttmlConverter extends Postprocessing { + private static final String TAG = "TttmlConverter"; + + TttmlConverter(DownloadMission mission) { + super(mission); + recommendedReserve = 0;// due how XmlPullParser works, the xml is fully loaded on the ram + worksOnSameFile = true; + } + + @Override + int process(SharpStream out, SharpStream... sources) throws IOException { + // check if the subtitle is already in srt and copy, this should never happen + String format = getArgumentAt(0, null); + + if (format == null || format.equals("ttml")) { + SubtitleConverter ttmlDumper = new SubtitleConverter(); + + try { + ttmlDumper.dumpTTML( + sources[0], + out, + getArgumentAt(1, "true").equals("true"), + getArgumentAt(2, "true").equals("true") + ); + } catch (Exception err) { + Log.e(TAG, "subtitle parse failed", err); + + if (err instanceof IOException) { + return 1; + } else if (err instanceof ParseException) { + return 2; + } else if (err instanceof SAXException) { + return 3; + } else if (err instanceof ParserConfigurationException) { + return 4; + } else if (err instanceof XPathExpressionException) { + return 7; + } + + return 8; + } + + return OK_RESULT; + } else if (format.equals("srt")) { + byte[] buffer = new byte[8 * 1024]; + int read; + while ((read = sources[0].read(buffer)) > 0) { + out.write(buffer, 0, read); + } + return OK_RESULT; + } + + throw new UnsupportedOperationException("Can't convert this subtitle, unimplemented format: " + format); + } + +} diff --git a/app/src/main/java/us/shandian/giga/postprocessing/WebMMuxer.java b/app/src/main/java/us/shandian/giga/postprocessing/WebMMuxer.java new file mode 100644 index 000000000..009a9a66b --- /dev/null +++ b/app/src/main/java/us/shandian/giga/postprocessing/WebMMuxer.java @@ -0,0 +1,44 @@ +package us.shandian.giga.postprocessing; + +import org.schabi.newpipe.streams.WebMReader.TrackKind; +import org.schabi.newpipe.streams.WebMReader.WebMTrack; +import org.schabi.newpipe.streams.WebMWriter; +import org.schabi.newpipe.streams.io.SharpStream; + +import java.io.IOException; + +import us.shandian.giga.get.DownloadMission; + +/** + * @author kapodamy + */ +class WebMMuxer extends Postprocessing { + + WebMMuxer(DownloadMission mission) { + super(mission); + recommendedReserve = 2048 * 1024;// 2 MiB + worksOnSameFile = true; + } + + @Override + int process(SharpStream out, SharpStream... sources) throws IOException { + WebMWriter muxer = new WebMWriter(sources); + muxer.parseSources(); + + // youtube uses a webm with a fake video track that acts as a "cover image" + WebMTrack[] tracks = muxer.getTracksFromSource(1); + int audioTrackIndex = 0; + for (int i = 0; i < tracks.length; i++) { + if (tracks[i].kind == TrackKind.Audio) { + audioTrackIndex = i; + break; + } + } + + muxer.selectTracks(0, audioTrackIndex); + muxer.build(out); + + return OK_RESULT; + } + +} diff --git a/app/src/main/java/us/shandian/giga/postprocessing/io/ChunkFileInputStream.java b/app/src/main/java/us/shandian/giga/postprocessing/io/ChunkFileInputStream.java new file mode 100644 index 000000000..cd62c5d22 --- /dev/null +++ b/app/src/main/java/us/shandian/giga/postprocessing/io/ChunkFileInputStream.java @@ -0,0 +1,153 @@ +package us.shandian.giga.postprocessing.io; + +import org.schabi.newpipe.streams.io.SharpStream; + +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; + +public class ChunkFileInputStream extends SharpStream { + + private RandomAccessFile source; + private final long offset; + private final long length; + private long position; + + public ChunkFileInputStream(File file, long start, long end, String mode) throws IOException { + source = new RandomAccessFile(file, mode); + offset = start; + length = end - start; + position = 0; + + if (length < 1) { + source.close(); + throw new IOException("The chunk is empty or invalid"); + } + if (source.length() < end) { + try { + throw new IOException(String.format("invalid file length. expected = %s found = %s", end, source.length())); + } finally { + source.close(); + } + } + + source.seek(offset); + } + + /** + * Get absolute position on file + * + * @return the position + */ + public long getFilePointer() { + return offset + position; + } + + @Override + public int read() throws IOException { + if ((position + 1) > length) { + return 0; + } + + int res = source.read(); + if (res >= 0) { + position++; + } + + return res; + } + + @Override + public int read(byte b[]) throws IOException { + return read(b, 0, b.length); + } + + @Override + public int read(byte b[], int off, int len) throws IOException { + if ((position + len) > length) { + len = (int) (length - position); + } + if (len == 0) { + return 0; + } + + int res = source.read(b, off, len); + position += res; + + return res; + } + + @Override + public long skip(long pos) throws IOException { + pos = Math.min(pos + position, length); + + if (pos == 0) { + return 0; + } + + source.seek(offset + pos); + + long oldPos = position; + position = pos; + + return pos - oldPos; + } + + @Override + public int available() { + return (int) (length - position); + } + + @SuppressWarnings("EmptyCatchBlock") + @Override + public void dispose() { + try { + source.close(); + } catch (IOException err) { + } finally { + source = null; + } + } + + @Override + public boolean isDisposed() { + return source == null; + } + + @Override + public void rewind() throws IOException { + position = 0; + source.seek(offset); + } + + @Override + public boolean canRewind() { + return true; + } + + @Override + public boolean canRead() { + return true; + } + + @Override + public boolean canWrite() { + return false; + } + + @Override + public void write(byte value) { + } + + @Override + public void write(byte[] buffer) { + } + + @Override + public void write(byte[] buffer, int offset, int count) { + } + + @Override + public void flush() { + } +} diff --git a/app/src/main/java/us/shandian/giga/postprocessing/io/CircularFile.java b/app/src/main/java/us/shandian/giga/postprocessing/io/CircularFile.java new file mode 100644 index 000000000..d2fc82d33 --- /dev/null +++ b/app/src/main/java/us/shandian/giga/postprocessing/io/CircularFile.java @@ -0,0 +1,375 @@ +package us.shandian.giga.postprocessing.io; + +import org.schabi.newpipe.streams.io.SharpStream; + +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.util.ArrayList; + +public class CircularFile extends SharpStream { + + private final static int AUX_BUFFER_SIZE = 1024 * 1024;// 1 MiB + private final static int AUX_BUFFER_SIZE2 = 512 * 1024;// 512 KiB + private final static int NOTIFY_BYTES_INTERVAL = 64 * 1024;// 64 KiB + private final static int QUEUE_BUFFER_SIZE = 8 * 1024;// 8 KiB + private final static boolean IMMEDIATE_AUX_BUFFER_FLUSH = false; + + private RandomAccessFile out; + private long position; + private long maxLengthKnown = -1; + + private ArrayList auxiliaryBuffers; + private OffsetChecker callback; + private ManagedBuffer queue; + private long startOffset; + private ProgressReport onProgress; + private long reportPosition; + + public CircularFile(File file, long offset, ProgressReport progressReport, OffsetChecker checker) throws IOException { + if (checker == null) { + throw new NullPointerException("checker is null"); + } + + try { + queue = new ManagedBuffer(QUEUE_BUFFER_SIZE); + out = new RandomAccessFile(file, "rw"); + out.seek(offset); + position = offset; + } catch (IOException err) { + try { + if (out != null) { + out.close(); + } + } catch (IOException e) { + // nothing to do + } + throw err; + } + + auxiliaryBuffers = new ArrayList<>(15); + callback = checker; + startOffset = offset; + reportPosition = offset; + onProgress = progressReport; + + } + + /** + * Close the file without flushing any buffer + */ + @Override + public void dispose() { + try { + auxiliaryBuffers = null; + if (out != null) { + out.close(); + out = null; + } + } catch (IOException err) { + // nothing to do + } + } + + /** + * Flush any buffer and close the output file. Use this method if the + * operation is successful + * + * @return the final length of the file + * @throws IOException if an I/O error occurs + */ + public long finalizeFile() throws IOException { + flushEverything(); + + if (maxLengthKnown > -1) { + position = maxLengthKnown; + } + if (position < out.length()) { + out.setLength(position); + } + + dispose(); + + return position; + } + + @Override + public void write(byte b) throws IOException { + write(new byte[]{b}, 0, 1); + } + + @Override + public void write(byte b[]) throws IOException { + write(b, 0, b.length); + } + + @Override + public void write(byte b[], int off, int len) throws IOException { + if (len == 0) { + return; + } + + long end = callback.check(); + long available; + + if (end == -1) { + available = Long.MAX_VALUE; + } else { + if (end < startOffset) { + throw new IOException("The reported offset is invalid. reported offset is " + String.valueOf(end)); + } + available = end - position; + } + + // Check if possible flush one or more auxiliary buffer + if (auxiliaryBuffers.size() > 0) { + ManagedBuffer aux = auxiliaryBuffers.get(0); + + // check if there is enough space to flush it completely + while (available >= (aux.size + queue.size)) { + available -= aux.size; + writeQueue(aux.buffer, 0, aux.size); + aux.dereference(); + auxiliaryBuffers.remove(0); + + if (auxiliaryBuffers.size() < 1) { + aux = null; + break; + } + aux = auxiliaryBuffers.get(0); + } + + if (IMMEDIATE_AUX_BUFFER_FLUSH) { + // try partial flush to avoid allocate another auxiliary buffer + if (aux != null && aux.available() < len && available > queue.size) { + int size = Math.min(aux.size, (int) available - queue.size); + + writeQueue(aux.buffer, 0, size); + aux.dereference(size); + + available -= size; + } + } + } + + if (auxiliaryBuffers.size() < 1 && available > (len + queue.size)) { + writeQueue(b, off, len); + } else { + int i = auxiliaryBuffers.size() - 1; + while (len > 0) { + if (i < 0) { + // allocate a new auxiliary buffer + auxiliaryBuffers.add(new ManagedBuffer(AUX_BUFFER_SIZE)); + i++; + } + + ManagedBuffer aux = auxiliaryBuffers.get(i); + available = aux.available(); + + if (available < 1) { + // secondary auxiliary buffer + available = len; + aux = new ManagedBuffer(Math.max(len, AUX_BUFFER_SIZE2)); + auxiliaryBuffers.add(aux); + i++; + } else { + available = Math.min(len, available); + } + + aux.write(b, off, (int) available); + + len -= available; + if (len > 0) off += available; + } + } + } + + private void writeOutside(byte buffer[], int offset, int length) throws IOException { + out.write(buffer, offset, length); + position += length; + + if (onProgress != null && position > reportPosition) { + reportPosition = position + NOTIFY_BYTES_INTERVAL; + onProgress.report(position); + } + } + + private void writeQueue(byte[] buffer, int offset, int length) throws IOException { + while (length > 0) { + if (queue.available() < length) { + flushQueue(); + + if (length >= queue.buffer.length) { + writeOutside(buffer, offset, length); + return; + } + } + + int size = Math.min(queue.available(), length); + queue.write(buffer, offset, size); + + offset += size; + length -= size; + } + + if (queue.size >= queue.buffer.length) { + flushQueue(); + } + } + + private void flushQueue() throws IOException { + writeOutside(queue.buffer, 0, queue.size); + queue.size = 0; + } + + private void flushEverything() throws IOException { + flushQueue(); + + if (auxiliaryBuffers.size() > 0) { + for (ManagedBuffer aux : auxiliaryBuffers) { + writeOutside(aux.buffer, 0, aux.size); + aux.dereference(); + } + auxiliaryBuffers.clear(); + } + } + + /** + * Flush any buffer directly to the file. Warning: use this method ONLY if + * all read dependencies are disposed + * + * @throws IOException if the dependencies are not disposed + */ + @Override + public void flush() throws IOException { + if (callback.check() != -1) { + throw new IOException("All read dependencies of this file must be disposed first"); + } + flushEverything(); + + // Save the current file length in case the method {@code rewind()} is called + if (position > maxLengthKnown) { + maxLengthKnown = position; + } + } + + @Override + public void rewind() throws IOException { + flush(); + out.seek(startOffset); + + if (onProgress != null) { + onProgress.report(-position); + } + + position = startOffset; + reportPosition = startOffset; + + } + + @Override + public long skip(long amount) throws IOException { + flush(); + position += amount; + + out.seek(position); + + return amount; + } + + @Override + public boolean isDisposed() { + return out == null; + } + + @Override + public boolean canRewind() { + return true; + } + + @Override + public boolean canWrite() { + return true; + } + + // + @Override + public boolean canRead() { + return false; + } + + @Override + public int read() { + throw new UnsupportedOperationException("write-only"); + } + + @Override + public int read(byte[] buffer) { + throw new UnsupportedOperationException("write-only"); + } + + @Override + public int read(byte[] buffer, int offset, int count) { + throw new UnsupportedOperationException("write-only"); + } + + @Override + public int available() { + throw new UnsupportedOperationException("write-only"); + } +// + + public interface OffsetChecker { + + /** + * Checks the amount of available space ahead + * + * @return absolute offset in the file where no more data SHOULD NOT be + * written. If the value is -1 the whole file will be used + */ + long check(); + } + + public interface ProgressReport { + + void report(long progress); + } + + class ManagedBuffer { + + byte[] buffer; + int size; + + ManagedBuffer(int length) { + buffer = new byte[length]; + } + + void dereference() { + buffer = null; + size = 0; + } + + void dereference(int amount) { + if (amount > size) { + throw new IndexOutOfBoundsException("Invalid dereference amount (" + amount + ">=" + size + ")"); + } + size -= amount; + System.arraycopy(buffer, amount, buffer, 0, size); + } + + protected int available() { + return buffer.length - size; + } + + private void write(byte[] b, int off, int len) { + System.arraycopy(b, off, buffer, size, len); + size += len; + } + + @Override + public String toString() { + return "holding: " + String.valueOf(size) + " length: " + String.valueOf(buffer.length) + " available: " + String.valueOf(available()); + } + + } +} diff --git a/app/src/main/java/us/shandian/giga/postprocessing/io/FileStream.java b/app/src/main/java/us/shandian/giga/postprocessing/io/FileStream.java new file mode 100644 index 000000000..c1b675eef --- /dev/null +++ b/app/src/main/java/us/shandian/giga/postprocessing/io/FileStream.java @@ -0,0 +1,126 @@ +package us.shandian.giga.postprocessing.io; + +import org.schabi.newpipe.streams.io.SharpStream; + +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.channels.FileChannel; + +/** + * @author kapodamy + */ +public class FileStream extends SharpStream { + + public enum Mode { + Read, + ReadWrite + } + + public RandomAccessFile source; + private final Mode mode; + + public FileStream(String path, Mode mode) throws IOException { + String flags; + + if (mode == Mode.Read) { + flags = "r"; + } else { + flags = "rw"; + } + + this.mode = mode; + source = new RandomAccessFile(path, flags); + } + + @Override + public int read() throws IOException { + return source.read(); + } + + @Override + public int read(byte b[]) throws IOException { + return read(b, 0, b.length); + } + + @Override + public int read(byte b[], int off, int len) throws IOException { + return source.read(b, off, len); + } + + @Override + public long skip(long pos) throws IOException { + FileChannel fc = source.getChannel(); + fc.position(fc.position() + pos); + return pos; + } + + @Override + public int available() { + try { + return (int) (source.length() - source.getFilePointer()); + } catch (IOException ex) { + return 0; + } + } + + @SuppressWarnings("EmptyCatchBlock") + @Override + public void dispose() { + try { + source.close(); + } catch (IOException err) { + + } finally { + source = null; + } + } + + @Override + public boolean isDisposed() { + return source == null; + } + + @Override + public void rewind() throws IOException { + source.getChannel().position(0); + } + + @Override + public boolean canRewind() { + return true; + } + + @Override + public boolean canRead() { + return mode == Mode.Read || mode == Mode.ReadWrite; + } + + @Override + public boolean canWrite() { + return mode == Mode.ReadWrite; + } + + @Override + public void write(byte value) throws IOException { + source.write(value); + } + + @Override + public void write(byte[] buffer) throws IOException { + source.write(buffer); + } + + @Override + public void write(byte[] buffer, int offset, int count) throws IOException { + source.write(buffer, offset, count); + } + + @Override + public void flush() { + } + + @Override + public void setLength(long length) throws IOException { + source.setLength(length); + } +} diff --git a/app/src/main/java/us/shandian/giga/postprocessing/io/SharpInputStream.java b/app/src/main/java/us/shandian/giga/postprocessing/io/SharpInputStream.java new file mode 100644 index 000000000..52e0775da --- /dev/null +++ b/app/src/main/java/us/shandian/giga/postprocessing/io/SharpInputStream.java @@ -0,0 +1,59 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ +package us.shandian.giga.postprocessing.io; + +import android.support.annotation.NonNull; + +import org.schabi.newpipe.streams.io.SharpStream; + +import java.io.IOException; +import java.io.InputStream; + +/** + * Wrapper for the classic {@link java.io.InputStream} + * @author kapodamy + */ +public class SharpInputStream extends InputStream { + + private final SharpStream base; + + public SharpInputStream(SharpStream base) throws IOException { + if (!base.canRead()) { + throw new IOException("The provided stream is not readable"); + } + this.base = base; + } + + @Override + public int read() throws IOException { + return base.read(); + } + + @Override + public int read(@NonNull byte[] bytes) throws IOException { + return base.read(bytes); + } + + @Override + public int read(@NonNull byte[] bytes, int i, int i1) throws IOException { + return base.read(bytes, i, i1); + } + + @Override + public long skip(long l) throws IOException { + return base.skip(l); + } + + @Override + public int available() { + return base.available(); + } + + @Override + public void close() { + base.dispose(); + } +} diff --git a/app/src/main/java/us/shandian/giga/service/DownloadManager.java b/app/src/main/java/us/shandian/giga/service/DownloadManager.java new file mode 100644 index 000000000..6bcf84745 --- /dev/null +++ b/app/src/main/java/us/shandian/giga/service/DownloadManager.java @@ -0,0 +1,676 @@ +package us.shandian.giga.service; + +import android.content.Context; +import android.os.Handler; +import android.support.annotation.NonNull; +import android.support.annotation.Nullable; +import android.support.v7.util.DiffUtil; +import android.util.Log; +import android.widget.Toast; + +import org.schabi.newpipe.R; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; + +import us.shandian.giga.get.DownloadMission; +import us.shandian.giga.get.FinishedMission; +import us.shandian.giga.get.Mission; +import us.shandian.giga.get.sqlite.DownloadDataSource; +import us.shandian.giga.util.Utility; + +import static org.schabi.newpipe.BuildConfig.DEBUG; + +public class DownloadManager { + private static final String TAG = DownloadManager.class.getSimpleName(); + + enum NetworkState {Unavailable, WifiOperating, MobileOperating, OtherOperating} + + public final static int SPECIAL_NOTHING = 0; + public final static int SPECIAL_PENDING = 1; + public final static int SPECIAL_FINISHED = 2; + + private final DownloadDataSource mDownloadDataSource; + + private final ArrayList mMissionsPending = new ArrayList<>(); + private final ArrayList mMissionsFinished; + + private final Handler mHandler; + private final File mPendingMissionsDir; + + private NetworkState mLastNetworkStatus = NetworkState.Unavailable; + + int mPrefMaxRetry; + boolean mPrefCrossNetwork; + + /** + * Create a new instance + * + * @param context Context for the data source for finished downloads + * @param handler Thread required for Messaging + */ + DownloadManager(@NonNull Context context, Handler handler) { + if (DEBUG) { + Log.d(TAG, "new DownloadManager instance. 0x" + Integer.toHexString(this.hashCode())); + } + + mDownloadDataSource = new DownloadDataSource(context); + mHandler = handler; + mMissionsFinished = loadFinishedMissions(); + mPendingMissionsDir = getPendingDir(context); + + if (!Utility.mkdir(mPendingMissionsDir, false)) { + throw new RuntimeException("failed to create pending_downloads in data directory"); + } + + loadPendingMissions(); + } + + private static File getPendingDir(@NonNull Context context) { + //File dir = new File(ContextCompat.getDataDir(context), "pending_downloads"); + File dir = context.getExternalFilesDir("pending_downloads"); + + if (dir == null) { + // One of the following paths are not accessible ¿unmounted internal memory? + // /storage/emulated/0/Android/data/org.schabi.newpipe[.debug]/pending_downloads + // /sdcard/Android/data/org.schabi.newpipe[.debug]/pending_downloads + Log.w(TAG, "path to pending downloads are not accessible"); + } + + return dir; + } + + /** + * Loads finished missions from the data source + */ + private ArrayList loadFinishedMissions() { + ArrayList finishedMissions = mDownloadDataSource.loadFinishedMissions(); + + // missions always is stored by creation order, simply reverse the list + ArrayList result = new ArrayList<>(finishedMissions.size()); + for (int i = finishedMissions.size() - 1; i >= 0; i--) { + FinishedMission mission = finishedMissions.get(i); + File file = mission.getDownloadedFile(); + + if (!file.isFile()) { + if (DEBUG) { + Log.d(TAG, "downloaded file removed: " + file.getAbsolutePath()); + } + mDownloadDataSource.deleteMission(mission); + continue; + } + + result.add(mission); + } + + return result; + } + + private void loadPendingMissions() { + File[] subs = mPendingMissionsDir.listFiles(); + + if (subs == null) { + Log.e(TAG, "listFiles() returned null"); + return; + } + if (subs.length < 1) { + return; + } + if (DEBUG) { + Log.d(TAG, "Loading pending downloads from directory: " + mPendingMissionsDir.getAbsolutePath()); + } + + for (File sub : subs) { + if (sub.isFile()) { + DownloadMission mis = Utility.readFromFile(sub); + + if (mis == null) { + //noinspection ResultOfMethodCallIgnored + sub.delete(); + } else { + if (mis.isFinished()) { + //noinspection ResultOfMethodCallIgnored + sub.delete(); + continue; + } + + File dl = mis.getDownloadedFile(); + boolean exists = dl.exists(); + + if (mis.postprocessingRunning && mis.postprocessingThis) { + // Incomplete post-processing results in a corrupted download file + // because the selected algorithm works on the same file to save space. + if (!dl.delete()) { + Log.w(TAG, "Unable to delete incomplete download file: " + sub.getPath()); + } + exists = true; + mis.postprocessingRunning = false; + mis.errCode = DownloadMission.ERROR_POSTPROCESSING_FAILED; + mis.errObject = new RuntimeException("stopped unexpectedly"); + } else if (exists && !dl.isFile()) { + // probably a folder, this should never happens + if (!sub.delete()) { + Log.w(TAG, "Unable to delete serialized file: " + sub.getPath()); + } + continue; + } + + if (!exists) { + // downloaded file deleted, reset mission state + DownloadMission m = new DownloadMission(mis.urls, mis.name, mis.location, mis.kind, mis.postprocessingName, mis.postprocessingArgs); + m.timestamp = mis.timestamp; + m.threadCount = mis.threadCount; + m.source = mis.source; + m.maxRetry = mis.maxRetry; + m.nearLength = mis.nearLength; + mis = m; + } + + mis.running = false; + mis.recovered = exists; + mis.metadata = sub; + mis.mHandler = mHandler; + + mMissionsPending.add(mis); + } + } + } + + if (mMissionsPending.size() > 1) { + Collections.sort(mMissionsPending, (mission1, mission2) -> Long.compare(mission1.timestamp, mission2.timestamp)); + } + } + + /** + * Start a new download mission + * + * @param urls the list of urls to download + * @param location the location + * @param name the name of the file to create + * @param kind type of file (a: audio v: video s: subtitle ?: file-extension defined) + * @param threads the number of threads maximal used to download chunks of the file. + * @param psName the name of the required post-processing algorithm, or {@code null} to ignore. + * @param source source url of the resource + * @param psArgs the arguments for the post-processing algorithm. + */ + void startMission(String[] urls, String location, String name, char kind, int threads, + String source, String psName, String[] psArgs, long nearLength) { + synchronized (this) { + // check for existing pending download + DownloadMission pendingMission = getPendingMission(location, name); + if (pendingMission != null) { + // generate unique filename (?) + try { + name = generateUniqueName(location, name); + } catch (Exception e) { + Log.e(TAG, "Unable to generate unique name", e); + name = System.currentTimeMillis() + name; + Log.i(TAG, "Using " + name); + } + } else { + // check for existing finished download + int index = getFinishedMissionIndex(location, name); + if (index >= 0) mDownloadDataSource.deleteMission(mMissionsFinished.remove(index)); + } + + DownloadMission mission = new DownloadMission(urls, name, location, kind, psName, psArgs); + mission.timestamp = System.currentTimeMillis(); + mission.threadCount = threads; + mission.source = source; + mission.mHandler = mHandler; + mission.maxRetry = mPrefMaxRetry; + mission.nearLength = nearLength; + + while (true) { + mission.metadata = new File(mPendingMissionsDir, String.valueOf(mission.timestamp)); + if (!mission.metadata.isFile() && !mission.metadata.exists()) { + try { + if (!mission.metadata.createNewFile()) + throw new RuntimeException("Cant create download metadata file"); + } catch (IOException e) { + throw new RuntimeException(e); + } + break; + } + mission.timestamp = System.currentTimeMillis(); + } + + mMissionsPending.add(mission); + + // Before starting, save the state in case the internet connection is not available + Utility.writeToFile(mission.metadata, mission); + + if (canDownloadInCurrentNetwork() && (getRunningMissionsCount() < 1)) { + mission.start(); + mHandler.sendEmptyMessage(DownloadManagerService.MESSAGE_RUNNING); + } + } + } + + + public void resumeMission(DownloadMission mission) { + if (!mission.running) { + mission.start(); + mHandler.sendEmptyMessage(DownloadManagerService.MESSAGE_RUNNING); + } + } + + public void pauseMission(DownloadMission mission) { + if (mission.running) { + mission.pause(); + mHandler.sendEmptyMessage(DownloadManagerService.MESSAGE_PAUSED); + } + } + + public void deleteMission(Mission mission) { + synchronized (this) { + if (mission instanceof DownloadMission) { + mMissionsPending.remove(mission); + } else if (mission instanceof FinishedMission) { + mMissionsFinished.remove(mission); + mDownloadDataSource.deleteMission(mission); + } + + mHandler.sendEmptyMessage(DownloadManagerService.MESSAGE_DELETED); + mission.delete(); + } + } + + + /** + * Get a pending mission by its location and name + * + * @param location the location + * @param name the name + * @return the mission or null if no such mission exists + */ + @Nullable + private DownloadMission getPendingMission(String location, String name) { + for (DownloadMission mission : mMissionsPending) { + if (location.equalsIgnoreCase(mission.location) && name.equalsIgnoreCase(mission.name)) { + return mission; + } + } + return null; + } + + /** + * Get a finished mission by its location and name + * + * @param location the location + * @param name the name + * @return the mission index or -1 if no such mission exists + */ + private int getFinishedMissionIndex(String location, String name) { + for (int i = 0; i < mMissionsFinished.size(); i++) { + FinishedMission mission = mMissionsFinished.get(i); + if (location.equalsIgnoreCase(mission.location) && name.equalsIgnoreCase(mission.name)) { + return i; + } + } + + return -1; + } + + public Mission getAnyMission(String location, String name) { + synchronized (this) { + Mission mission = getPendingMission(location, name); + if (mission != null) return mission; + + int idx = getFinishedMissionIndex(location, name); + if (idx >= 0) return mMissionsFinished.get(idx); + } + + return null; + } + + int getRunningMissionsCount() { + int count = 0; + synchronized (this) { + for (DownloadMission mission : mMissionsPending) { + if (mission.running && mission.errCode != DownloadMission.ERROR_POSTPROCESSING_FAILED && !mission.isFinished()) + count++; + } + } + + return count; + } + + void pauseAllMissions() { + synchronized (this) { + for (DownloadMission mission : mMissionsPending) mission.pause(); + } + } + + + /** + * Splits the filename into name and extension + *

+ * Dots are ignored if they appear: not at all, at the beginning of the file, + * at the end of the file + * + * @param name the name to split + * @return a string array with a length of 2 containing the name and the extension + */ + private static String[] splitName(String name) { + int dotIndex = name.lastIndexOf('.'); + if (dotIndex <= 0 || (dotIndex == name.length() - 1)) { + return new String[]{name, ""}; + } else { + return new String[]{name.substring(0, dotIndex), name.substring(dotIndex + 1)}; + } + } + + /** + * Generates a unique file name. + *

+ * e.g. "myName (1).txt" if the name "myName.txt" exists. + * + * @param location the location (to check for existing files) + * @param name the name of the file + * @return the unique file name + * @throws IllegalArgumentException if the location is not a directory + * @throws SecurityException if the location is not readable + */ + private static String generateUniqueName(String location, String name) { + if (location == null) throw new NullPointerException("location is null"); + if (name == null) throw new NullPointerException("name is null"); + File destination = new File(location); + if (!destination.isDirectory()) { + throw new IllegalArgumentException("location is not a directory: " + location); + } + final String[] nameParts = splitName(name); + String[] existingName = destination.list((dir, name1) -> name1.startsWith(nameParts[0])); + Arrays.sort(existingName); + String newName; + int downloadIndex = 0; + do { + newName = nameParts[0] + " (" + downloadIndex + ")." + nameParts[1]; + ++downloadIndex; + if (downloadIndex == 1000) { // Probably an error on our side + throw new RuntimeException("Too many existing files"); + } + } while (Arrays.binarySearch(existingName, newName) >= 0); + return newName; + } + + /** + * Set a pending download as finished + * + * @param mission the desired mission + */ + void setFinished(DownloadMission mission) { + synchronized (this) { + mMissionsPending.remove(mission); + mMissionsFinished.add(0, new FinishedMission(mission)); + mDownloadDataSource.addMission(mission); + } + } + + /** + * runs another mission in queue if possible + * + * @return true if exits pending missions running or a mission was started, otherwise, false + */ + boolean runAnotherMission() { + synchronized (this) { + if (mMissionsPending.size() < 1) return false; + + int i = getRunningMissionsCount(); + if (i > 0) return true; + + if (!canDownloadInCurrentNetwork()) return false; + + for (DownloadMission mission : mMissionsPending) { + if (!mission.running && mission.errCode == DownloadMission.ERROR_NOTHING && mission.enqueued) { + resumeMission(mission); + return true; + } + } + + return false; + } + } + + public MissionIterator getIterator() { + return new MissionIterator(); + } + + /** + * Forget all finished downloads, but, doesn't delete any file + */ + public void forgetFinishedDownloads() { + synchronized (this) { + for (FinishedMission mission : mMissionsFinished) { + mDownloadDataSource.deleteMission(mission); + } + mMissionsFinished.clear(); + } + } + + private boolean canDownloadInCurrentNetwork() { + if (mLastNetworkStatus == NetworkState.Unavailable) return false; + return !(mPrefCrossNetwork && mLastNetworkStatus == NetworkState.MobileOperating); + } + + void handleConnectivityChange(NetworkState currentStatus) { + if (currentStatus == mLastNetworkStatus) return; + + mLastNetworkStatus = currentStatus; + + if (currentStatus == NetworkState.Unavailable) { + return; + } else if (currentStatus != NetworkState.MobileOperating || !mPrefCrossNetwork) { + return; + } + + boolean flag = false; + synchronized (this) { + for (DownloadMission mission : mMissionsPending) { + if (mission.running && mission.isFinished() && !mission.postprocessingRunning) { + flag = true; + mission.pause(); + } + } + } + + if (flag) mHandler.sendEmptyMessage(DownloadManagerService.MESSAGE_PAUSED); + } + + void updateMaximumAttempts() { + synchronized (this) { + for (DownloadMission mission : mMissionsPending) mission.maxRetry = mPrefMaxRetry; + } + } + + /** + * Fast check for pending downloads. If exists, the user will be notified + * TODO: call this method in somewhere + * + * @param context the application context + */ + public static void notifyUserPendingDownloads(Context context) { + int pending = getPendingDir(context).list().length; + if (pending < 1) return; + + Toast.makeText(context, context.getString( + R.string.msg_pending_downloads, + String.valueOf(pending) + ), Toast.LENGTH_LONG).show(); + } + + void checkForRunningMission(String location, String name, DownloadManagerService.DMChecker check) { + boolean listed; + boolean finished = false; + + synchronized (this) { + DownloadMission mission = getPendingMission(location, name); + if (mission != null) { + listed = true; + } else { + listed = getFinishedMissionIndex(location, name) >= 0; + finished = listed; + } + } + + check.callback(listed, finished); + } + + public class MissionIterator extends DiffUtil.Callback { + final Object FINISHED = new Object(); + final Object PENDING = new Object(); + + ArrayList snapshot; + ArrayList current; + ArrayList hidden; + + private MissionIterator() { + hidden = new ArrayList<>(2); + current = null; + snapshot = getSpecialItems(); + } + + private ArrayList getSpecialItems() { + synchronized (DownloadManager.this) { + ArrayList pending = new ArrayList<>(mMissionsPending); + ArrayList finished = new ArrayList<>(mMissionsFinished); + ArrayList remove = new ArrayList<>(hidden); + + // hide missions (if required) + Iterator iterator = remove.iterator(); + while (iterator.hasNext()) { + Mission mission = iterator.next(); + if (pending.remove(mission) || finished.remove(mission)) iterator.remove(); + } + + int fakeTotal = pending.size(); + if (fakeTotal > 0) fakeTotal++; + + fakeTotal += finished.size(); + if (finished.size() > 0) fakeTotal++; + + ArrayList list = new ArrayList<>(fakeTotal); + if (pending.size() > 0) { + list.add(PENDING); + list.addAll(pending); + } + if (finished.size() > 0) { + list.add(FINISHED); + list.addAll(finished); + } + + + return list; + } + } + + public MissionItem getItem(int position) { + Object object = snapshot.get(position); + + if (object == PENDING) return new MissionItem(SPECIAL_PENDING); + if (object == FINISHED) return new MissionItem(SPECIAL_FINISHED); + + return new MissionItem(SPECIAL_NOTHING, (Mission) object); + } + + public int getSpecialAtItem(int position) { + Object object = snapshot.get(position); + + if (object == PENDING) return SPECIAL_PENDING; + if (object == FINISHED) return SPECIAL_FINISHED; + + return SPECIAL_NOTHING; + } + + public MissionItem getItemUnsafe(int position) { + synchronized (DownloadManager.this) { + int count = mMissionsPending.size(); + int count2 = mMissionsFinished.size(); + + if (count > 0) { + position--; + if (position == -1) + return new MissionItem(SPECIAL_PENDING); + else if (position < count) + return new MissionItem(SPECIAL_NOTHING, mMissionsPending.get(position)); + else if (position == count && count2 > 0) + return new MissionItem(SPECIAL_FINISHED); + else + position -= count; + } else { + if (count2 > 0 && position == 0) { + return new MissionItem(SPECIAL_FINISHED); + } + } + + position--; + + if (count2 < 1) { + throw new RuntimeException( + String.format("Out of range. pending_count=%s finished_count=%s position=%s", count, count2, position) + ); + } + + return new MissionItem(SPECIAL_NOTHING, mMissionsFinished.get(position)); + } + } + + + public void start() { + current = getSpecialItems(); + } + + public void end() { + snapshot = current; + current = null; + } + + public void hide(Mission mission) { + hidden.add(mission); + } + + public void unHide(Mission mission) { + hidden.remove(mission); + } + + + @Override + public int getOldListSize() { + return snapshot.size(); + } + + @Override + public int getNewListSize() { + return current.size(); + } + + @Override + public boolean areItemsTheSame(int oldItemPosition, int newItemPosition) { + return snapshot.get(oldItemPosition) == current.get(newItemPosition); + } + + @Override + public boolean areContentsTheSame(int oldItemPosition, int newItemPosition) { + return areItemsTheSame(oldItemPosition, newItemPosition); + } + } + + public class MissionItem { + public int special; + public Mission mission; + + MissionItem(int s, Mission m) { + special = s; + mission = m; + } + + MissionItem(int s) { + this(s, null); + } + } + +} diff --git a/app/src/main/java/us/shandian/giga/service/DownloadManagerService.java b/app/src/main/java/us/shandian/giga/service/DownloadManagerService.java index ff410a79a..a57fe1734 100755 --- a/app/src/main/java/us/shandian/giga/service/DownloadManagerService.java +++ b/app/src/main/java/us/shandian/giga/service/DownloadManagerService.java @@ -2,67 +2,114 @@ package us.shandian.giga.service; import android.Manifest; import android.app.Notification; +import android.app.NotificationManager; import android.app.PendingIntent; import android.app.Service; +import android.content.BroadcastReceiver; +import android.content.ComponentName; import android.content.Context; import android.content.Intent; +import android.content.IntentFilter; +import android.content.ServiceConnection; +import android.content.SharedPreferences; import android.graphics.Bitmap; import android.graphics.BitmapFactory; +import android.net.ConnectivityManager; +import android.net.NetworkInfo; import android.net.Uri; import android.os.Binder; +import android.os.Build; import android.os.Handler; -import android.os.HandlerThread; import android.os.IBinder; +import android.os.Looper; import android.os.Message; +import android.preference.PreferenceManager; +import android.support.v4.app.NotificationCompat; import android.support.v4.app.NotificationCompat.Builder; import android.support.v4.content.PermissionChecker; import android.util.Log; +import android.util.SparseArray; import android.widget.Toast; import org.schabi.newpipe.R; import org.schabi.newpipe.download.DownloadActivity; -import org.schabi.newpipe.settings.NewPipeSettings; +import org.schabi.newpipe.player.helper.LockManager; +import java.io.File; import java.util.ArrayList; -import us.shandian.giga.get.DownloadDataSource; -import us.shandian.giga.get.DownloadManager; -import us.shandian.giga.get.DownloadManagerImpl; import us.shandian.giga.get.DownloadMission; -import us.shandian.giga.get.sqlite.SQLiteDownloadDataSource; +import us.shandian.giga.service.DownloadManager.NetworkState; +import static org.schabi.newpipe.BuildConfig.APPLICATION_ID; import static org.schabi.newpipe.BuildConfig.DEBUG; public class DownloadManagerService extends Service { - private static final String TAG = DownloadManagerService.class.getSimpleName(); + private static final String TAG = "DownloadManagerService"; - /** - * Message code of update messages stored as {@link Message#what}. - */ - private static final int UPDATE_MESSAGE = 0; - private static final int NOTIFICATION_ID = 1000; + public static final int MESSAGE_RUNNING = 0; + public static final int MESSAGE_PAUSED = 1; + public static final int MESSAGE_FINISHED = 2; + public static final int MESSAGE_PROGRESS = 3; + public static final int MESSAGE_ERROR = 4; + public static final int MESSAGE_DELETED = 5; + + private static final int FOREGROUND_NOTIFICATION_ID = 1000; + private static final int DOWNLOADS_NOTIFICATION_ID = 1001; + + private static final String EXTRA_URLS = "DownloadManagerService.extra.urls"; private static final String EXTRA_NAME = "DownloadManagerService.extra.name"; private static final String EXTRA_LOCATION = "DownloadManagerService.extra.location"; - private static final String EXTRA_IS_AUDIO = "DownloadManagerService.extra.is_audio"; + private static final String EXTRA_KIND = "DownloadManagerService.extra.kind"; private static final String EXTRA_THREADS = "DownloadManagerService.extra.threads"; + private static final String EXTRA_POSTPROCESSING_NAME = "DownloadManagerService.extra.postprocessingName"; + private static final String EXTRA_POSTPROCESSING_ARGS = "DownloadManagerService.extra.postprocessingArgs"; + private static final String EXTRA_SOURCE = "DownloadManagerService.extra.source"; + private static final String EXTRA_NEAR_LENGTH = "DownloadManagerService.extra.nearLength"; + private static final String ACTION_RESET_DOWNLOAD_FINISHED = APPLICATION_ID + ".reset_download_finished"; + private static final String ACTION_OPEN_DOWNLOADS_FINISHED = APPLICATION_ID + ".open_downloads_finished"; private DMBinder mBinder; private DownloadManager mManager; private Notification mNotification; private Handler mHandler; - private long mLastTimeStamp = System.currentTimeMillis(); - private DownloadDataSource mDataSource; + private boolean mForeground = false; + private NotificationManager notificationManager = null; + private boolean mDownloadNotificationEnable = true; + private int downloadDoneCount = 0; + private Builder downloadDoneNotification = null; + private StringBuilder downloadDoneList = null; - private final MissionListener missionListener = new MissionListener(); + private final ArrayList mEchoObservers = new ArrayList<>(1); + private BroadcastReceiver mNetworkStateListener; - private void notifyMediaScanner(DownloadMission mission) { - Uri uri = Uri.parse("file://" + mission.location + "/" + mission.name); - // notify media scanner on downloaded media file ... - sendBroadcast(new Intent(Intent.ACTION_MEDIA_SCANNER_SCAN_FILE, uri)); + private SharedPreferences mPrefs = null; + private final SharedPreferences.OnSharedPreferenceChangeListener mPrefChangeListener = this::handlePreferenceChange; + + private boolean mLockAcquired = false; + private LockManager mLock = null; + + private int downloadFailedNotificationID = DOWNLOADS_NOTIFICATION_ID + 1; + private Builder downloadFailedNotification = null; + private SparseArray mFailedDownloads = new SparseArray<>(5); + + private Bitmap icLauncher; + private Bitmap icDownloadDone; + private Bitmap icDownloadFailed; + + private PendingIntent mOpenDownloadList; + + /** + * notify media scanner on downloaded media file ... + * + * @param file the downloaded file + */ + private void notifyMediaScanner(File file) { + sendBroadcast(new Intent(Intent.ACTION_MEDIA_SCANNER_SCAN_FILE, Uri.fromFile(file))); } @Override @@ -74,87 +121,92 @@ public class DownloadManagerService extends Service { } mBinder = new DMBinder(); - if (mDataSource == null) { - mDataSource = new SQLiteDownloadDataSource(this); - } - if (mManager == null) { - ArrayList paths = new ArrayList<>(2); - paths.add(NewPipeSettings.getVideoDownloadPath(this)); - paths.add(NewPipeSettings.getAudioDownloadPath(this)); - mManager = new DownloadManagerImpl(paths, mDataSource, this); - if (DEBUG) { - Log.d(TAG, "mManager == null"); - Log.d(TAG, "Download directory: " + paths); + mHandler = new Handler(Looper.myLooper()) { + @Override + public void handleMessage(Message msg) { + DownloadManagerService.this.handleMessage(msg); } - } + }; + + mManager = new DownloadManager(this, mHandler); Intent openDownloadListIntent = new Intent(this, DownloadActivity.class) .setAction(Intent.ACTION_MAIN); - PendingIntent pendingIntent = PendingIntent.getActivity(this, 0, + mOpenDownloadList = PendingIntent.getActivity(this, 0, openDownloadListIntent, PendingIntent.FLAG_UPDATE_CURRENT); - Bitmap iconBitmap = BitmapFactory.decodeResource(this.getResources(), R.mipmap.ic_launcher); + icLauncher = BitmapFactory.decodeResource(this.getResources(), R.mipmap.ic_launcher); Builder builder = new Builder(this, getString(R.string.notification_channel_id)) - .setContentIntent(pendingIntent) + .setContentIntent(mOpenDownloadList) .setSmallIcon(android.R.drawable.stat_sys_download) - .setLargeIcon(iconBitmap) + .setLargeIcon(icLauncher) .setContentTitle(getString(R.string.msg_running)) .setContentText(getString(R.string.msg_running_detail)); mNotification = builder.build(); + notificationManager = (NotificationManager) getSystemService(Context.NOTIFICATION_SERVICE); - HandlerThread thread = new HandlerThread("ServiceMessenger"); - thread.start(); - - mHandler = new Handler(thread.getLooper()) { + mNetworkStateListener = new BroadcastReceiver() { @Override - public void handleMessage(Message msg) { - switch (msg.what) { - case UPDATE_MESSAGE: { - int runningCount = 0; - - for (int i = 0; i < mManager.getCount(); i++) { - if (mManager.getMission(i).running) { - runningCount++; - } - } - updateState(runningCount); - break; - } + public void onReceive(Context context, Intent intent) { + if (intent.getBooleanExtra(ConnectivityManager.EXTRA_NO_CONNECTIVITY, false)) { + handleConnectivityChange(null); + return; } + handleConnectivityChange(intent.getParcelableExtra(ConnectivityManager.EXTRA_NETWORK_INFO)); } }; + registerReceiver(mNetworkStateListener, new IntentFilter(ConnectivityManager.CONNECTIVITY_ACTION)); - } + mPrefs = PreferenceManager.getDefaultSharedPreferences(this); + mPrefs.registerOnSharedPreferenceChangeListener(mPrefChangeListener); - private void startMissionAsync(final String url, final String location, final String name, - final boolean isAudio, final int threads) { - mHandler.post(new Runnable() { - @Override - public void run() { - int missionId = mManager.startMission(url, location, name, isAudio, threads); - mBinder.onMissionAdded(mManager.getMission(missionId)); - } - }); + handlePreferenceChange(mPrefs, getString(R.string.downloads_cross_network)); + handlePreferenceChange(mPrefs, getString(R.string.downloads_maximum_retry)); + + mLock = new LockManager(this); } @Override public int onStartCommand(Intent intent, int flags, int startId) { if (DEBUG) { + if (intent == null) { + Log.d(TAG, "Restarting"); + return START_NOT_STICKY; + } Log.d(TAG, "Starting"); } Log.i(TAG, "Got intent: " + intent); String action = intent.getAction(); - if (action != null && action.equals(Intent.ACTION_RUN)) { - String name = intent.getStringExtra(EXTRA_NAME); - String location = intent.getStringExtra(EXTRA_LOCATION); - int threads = intent.getIntExtra(EXTRA_THREADS, 1); - boolean isAudio = intent.getBooleanExtra(EXTRA_IS_AUDIO, false); - String url = intent.getDataString(); - startMissionAsync(url, location, name, isAudio, threads); + if (action != null) { + if (action.equals(Intent.ACTION_RUN)) { + String[] urls = intent.getStringArrayExtra(EXTRA_URLS); + String name = intent.getStringExtra(EXTRA_NAME); + String location = intent.getStringExtra(EXTRA_LOCATION); + int threads = intent.getIntExtra(EXTRA_THREADS, 1); + char kind = intent.getCharExtra(EXTRA_KIND, '?'); + String psName = intent.getStringExtra(EXTRA_POSTPROCESSING_NAME); + String[] psArgs = intent.getStringArrayExtra(EXTRA_POSTPROCESSING_ARGS); + String source = intent.getStringExtra(EXTRA_SOURCE); + long nearLength = intent.getLongExtra(EXTRA_NEAR_LENGTH, 0); + + mHandler.post(() -> mManager.startMission(urls, location, name, kind, threads, source, psName, psArgs, nearLength)); + + } else if (downloadDoneNotification != null) { + if (action.equals(ACTION_RESET_DOWNLOAD_FINISHED) || action.equals(ACTION_OPEN_DOWNLOADS_FINISHED)) { + downloadDoneCount = 0; + downloadDoneList.setLength(0); + } + if (action.equals(ACTION_OPEN_DOWNLOADS_FINISHED)) { + startActivity(new Intent(this, DownloadActivity.class) + .setAction(Intent.ACTION_MAIN) + .addFlags(Intent.FLAG_ACTIVITY_NEW_TASK) + ); + } + } } return START_NOT_STICKY; } @@ -167,11 +219,23 @@ public class DownloadManagerService extends Service { Log.d(TAG, "Destroying"); } - for (int i = 0; i < mManager.getCount(); i++) { - mManager.pauseMission(i); + stopForeground(true); + + if (notificationManager != null && downloadDoneNotification != null) { + downloadDoneNotification.setDeleteIntent(null);// prevent NewPipe running when is killed, cleared from recent, etc + notificationManager.notify(DOWNLOADS_NOTIFICATION_ID, downloadDoneNotification.build()); } - stopForeground(true); + mManager.pauseAllMissions(); + + manageLock(false); + + unregisterReceiver(mNetworkStateListener); + mPrefs.unregisterOnSharedPreferenceChangeListener(mPrefChangeListener); + + if (icDownloadDone != null) icDownloadDone.recycle(); + if (icDownloadFailed != null) icDownloadFailed.recycle(); + if (icLauncher != null) icLauncher.recycle(); } @Override @@ -192,53 +256,236 @@ public class DownloadManagerService extends Service { return mBinder; } - private void postUpdateMessage() { - mHandler.sendEmptyMessage(UPDATE_MESSAGE); - } + public void handleMessage(Message msg) { + DownloadMission mission = (DownloadMission) msg.obj; - private void updateState(int runningCount) { - if (runningCount == 0) { - stopForeground(true); - } else { - startForeground(NOTIFICATION_ID, mNotification); + switch (msg.what) { + case MESSAGE_FINISHED: + notifyMediaScanner(mission.getDownloadedFile()); + notifyFinishedDownload(mission.name); + mManager.setFinished(mission); + updateForegroundState(mManager.runAnotherMission()); + break; + case MESSAGE_RUNNING: + case MESSAGE_PROGRESS: + updateForegroundState(true); + break; + case MESSAGE_ERROR: + notifyFailedDownload(mission); + updateForegroundState(mManager.runAnotherMission()); + break; + case MESSAGE_PAUSED: + updateForegroundState(mManager.getRunningMissionsCount() > 0); + break; + } + + if (msg.what != MESSAGE_ERROR) + mFailedDownloads.delete(mFailedDownloads.indexOfValue(mission)); + + synchronized (mEchoObservers) { + for (Handler handler : mEchoObservers) { + Message echo = new Message(); + echo.what = msg.what; + echo.obj = msg.obj; + + handler.sendMessage(echo); + } } } - public static void startMission(Context context, String url, String location, String name, boolean isAudio, int threads) { + private void handleConnectivityChange(NetworkInfo info) { + NetworkState status; + + if (info == null) { + status = NetworkState.Unavailable; + Log.i(TAG, "actual connectivity status is unavailable"); + } else if (!info.isAvailable() || !info.isConnected()) { + status = NetworkState.Unavailable; + Log.i(TAG, "actual connectivity status is not available and not connected"); + } else { + int type = info.getType(); + if (type == ConnectivityManager.TYPE_MOBILE || type == ConnectivityManager.TYPE_MOBILE_DUN) { + status = NetworkState.MobileOperating; + } else if (type == ConnectivityManager.TYPE_WIFI) { + status = NetworkState.WifiOperating; + } else if (type == ConnectivityManager.TYPE_WIMAX || + type == ConnectivityManager.TYPE_ETHERNET || + type == ConnectivityManager.TYPE_BLUETOOTH) { + status = NetworkState.OtherOperating; + } else { + status = NetworkState.Unavailable; + } + Log.i(TAG, "actual connectivity status is " + status.name()); + } + + if (mManager == null) return;// avoid race-conditions while the service is starting + mManager.handleConnectivityChange(status); + } + + private void handlePreferenceChange(SharedPreferences prefs, String key) { + if (key.equals(getString(R.string.downloads_maximum_retry))) { + try { + String value = prefs.getString(key, getString(R.string.downloads_maximum_retry_default)); + mManager.mPrefMaxRetry = Integer.parseInt(value); + } catch (Exception e) { + mManager.mPrefMaxRetry = 0; + } + mManager.updateMaximumAttempts(); + } else if (key.equals(getString(R.string.downloads_cross_network))) { + mManager.mPrefCrossNetwork = prefs.getBoolean(key, false); + } + } + + public void updateForegroundState(boolean state) { + if (state == mForeground) return; + + if (state) { + startForeground(FOREGROUND_NOTIFICATION_ID, mNotification); + } else { + stopForeground(true); + } + + manageLock(state); + + mForeground = state; + } + + public static void startMission(Context context, String urls[], String location, String name, char kind, + int threads, String source, String psName, String[] psArgs, long nearLength) { Intent intent = new Intent(context, DownloadManagerService.class); intent.setAction(Intent.ACTION_RUN); - intent.setData(Uri.parse(url)); + intent.putExtra(EXTRA_URLS, urls); intent.putExtra(EXTRA_NAME, name); intent.putExtra(EXTRA_LOCATION, location); - intent.putExtra(EXTRA_IS_AUDIO, isAudio); + intent.putExtra(EXTRA_KIND, kind); intent.putExtra(EXTRA_THREADS, threads); + intent.putExtra(EXTRA_SOURCE, source); + intent.putExtra(EXTRA_POSTPROCESSING_NAME, psName); + intent.putExtra(EXTRA_POSTPROCESSING_ARGS, psArgs); + intent.putExtra(EXTRA_NEAR_LENGTH, nearLength); context.startService(intent); } + public static void checkForRunningMission(Context context, String location, String name, DMChecker check) { + Intent intent = new Intent(); + intent.setClass(context, DownloadManagerService.class); + context.bindService(intent, new ServiceConnection() { + @Override + public void onServiceConnected(ComponentName cname, IBinder service) { + try { + ((DMBinder) service).getDownloadManager().checkForRunningMission(location, name, check); + } catch (Exception err) { + Log.w(TAG, "checkForRunningMission() callback is defective", err); + } - private class MissionListener implements DownloadMission.MissionListener { - @Override - public void onProgressUpdate(DownloadMission downloadMission, long done, long total) { - long now = System.currentTimeMillis(); - long delta = now - mLastTimeStamp; - if (delta > 2000) { - postUpdateMessage(); - mLastTimeStamp = now; + // TODO: find a efficient way to unbind the service. This destroy the service due idle, but is started again when the user start a download. + context.unbindService(this); } + + @Override + public void onServiceDisconnected(ComponentName name) { + } + }, Context.BIND_AUTO_CREATE); + } + + public void notifyFinishedDownload(String name) { + if (!mDownloadNotificationEnable || notificationManager == null) { + return; } - @Override - public void onFinish(DownloadMission downloadMission) { - postUpdateMessage(); - notifyMediaScanner(downloadMission); + if (downloadDoneNotification == null) { + downloadDoneList = new StringBuilder(name.length()); + + icDownloadDone = BitmapFactory.decodeResource(this.getResources(), android.R.drawable.stat_sys_download_done); + downloadDoneNotification = new Builder(this, getString(R.string.notification_channel_id)) + .setAutoCancel(true) + .setLargeIcon(icDownloadDone) + .setSmallIcon(android.R.drawable.stat_sys_download_done) + .setDeleteIntent(makePendingIntent(ACTION_RESET_DOWNLOAD_FINISHED)) + .setContentIntent(makePendingIntent(ACTION_OPEN_DOWNLOADS_FINISHED)); } - @Override - public void onError(DownloadMission downloadMission, int errCode) { - postUpdateMessage(); + if (downloadDoneCount < 1) { + downloadDoneList.append(name); + + if (android.os.Build.VERSION.SDK_INT < Build.VERSION_CODES.LOLLIPOP) { + downloadDoneNotification.setContentTitle(getString(R.string.app_name)); + } else { + downloadDoneNotification.setContentTitle(null); + } + + downloadDoneNotification.setContentText(getString(R.string.download_finished)); + downloadDoneNotification.setStyle(new NotificationCompat.BigTextStyle() + .setBigContentTitle(getString(R.string.download_finished)) + .bigText(name) + ); + } else { + downloadDoneList.append('\n'); + downloadDoneList.append(name); + + downloadDoneNotification.setStyle(new NotificationCompat.BigTextStyle().bigText(downloadDoneList)); + downloadDoneNotification.setContentTitle(getString(R.string.download_finished_more, String.valueOf(downloadDoneCount + 1))); + downloadDoneNotification.setContentText(downloadDoneList); + } + + notificationManager.notify(DOWNLOADS_NOTIFICATION_ID, downloadDoneNotification.build()); + downloadDoneCount++; + } + + public void notifyFailedDownload(DownloadMission mission) { + if (!mDownloadNotificationEnable || mFailedDownloads.indexOfValue(mission) >= 0) return; + + int id = downloadFailedNotificationID++; + mFailedDownloads.put(id, mission); + + if (downloadFailedNotification == null) { + icDownloadFailed = BitmapFactory.decodeResource(this.getResources(), android.R.drawable.stat_sys_warning); + downloadFailedNotification = new Builder(this, getString(R.string.notification_channel_id)) + .setAutoCancel(true) + .setLargeIcon(icDownloadFailed) + .setSmallIcon(android.R.drawable.stat_sys_warning) + .setContentIntent(mOpenDownloadList); + } + + if (android.os.Build.VERSION.SDK_INT < Build.VERSION_CODES.LOLLIPOP) { + downloadFailedNotification.setContentTitle(getString(R.string.app_name)); + downloadFailedNotification.setStyle(new NotificationCompat.BigTextStyle() + .bigText(getString(R.string.download_failed).concat(": ").concat(mission.name))); + } else { + downloadFailedNotification.setContentTitle(getString(R.string.download_failed)); + downloadFailedNotification.setContentText(mission.name); + downloadFailedNotification.setStyle(new NotificationCompat.BigTextStyle() + .bigText(mission.name)); + } + + notificationManager.notify(id, downloadFailedNotification.build()); + } + + private PendingIntent makePendingIntent(String action) { + Intent intent = new Intent(this, DownloadManagerService.class).setAction(action); + return PendingIntent.getService(this, intent.hashCode(), intent, PendingIntent.FLAG_UPDATE_CURRENT); + } + + private void manageObservers(Handler handler, boolean add) { + synchronized (mEchoObservers) { + if (add) { + mEchoObservers.add(handler); + } else { + mEchoObservers.remove(handler); + } } } + private void manageLock(boolean acquire) { + if (acquire == mLockAcquired) return; + + if (acquire) + mLock.acquireWifiAndCpu(); + else + mLock.releaseWifiAndCpu(); + + mLockAcquired = acquire; + } // Wrapper of DownloadManager public class DMBinder extends Binder { @@ -246,14 +493,38 @@ public class DownloadManagerService extends Service { return mManager; } - public void onMissionAdded(DownloadMission mission) { - mission.addListener(missionListener); - postUpdateMessage(); + public void addMissionEventListener(Handler handler) { + manageObservers(handler, true); } - public void onMissionRemoved(DownloadMission mission) { - mission.removeListener(missionListener); - postUpdateMessage(); + public void removeMissionEventListener(Handler handler) { + manageObservers(handler, false); } + + public void clearDownloadNotifications() { + if (notificationManager == null) return; + if (downloadDoneNotification != null) { + notificationManager.cancel(DOWNLOADS_NOTIFICATION_ID); + downloadDoneList.setLength(0); + downloadDoneCount = 0; + } + if (downloadFailedNotification != null) { + for (; downloadFailedNotificationID > DOWNLOADS_NOTIFICATION_ID; downloadFailedNotificationID--) { + notificationManager.cancel(downloadFailedNotificationID); + } + mFailedDownloads.clear(); + downloadFailedNotificationID++; + } + } + + public void enableNotifications(boolean enable) { + mDownloadNotificationEnable = enable; + } + } + + public interface DMChecker { + void callback(boolean listed, boolean finished); + } + } diff --git a/app/src/main/java/us/shandian/giga/ui/adapter/MissionAdapter.java b/app/src/main/java/us/shandian/giga/ui/adapter/MissionAdapter.java index bca8796b5..df5f9e429 100644 --- a/app/src/main/java/us/shandian/giga/ui/adapter/MissionAdapter.java +++ b/app/src/main/java/us/shandian/giga/ui/adapter/MissionAdapter.java @@ -1,5 +1,6 @@ package us.shandian.giga.ui.adapter; +import android.annotation.SuppressLint; import android.app.Activity; import android.app.ProgressDialog; import android.content.Context; @@ -7,12 +8,21 @@ import android.content.Intent; import android.net.Uri; import android.os.AsyncTask; import android.os.Build; +import android.os.Bundle; +import android.os.Handler; +import android.os.Looper; +import android.os.Message; import android.support.annotation.NonNull; import android.support.annotation.Nullable; import android.support.v4.content.FileProvider; import android.support.v4.view.ViewCompat; +import android.support.v7.app.AlertDialog; +import android.support.v7.util.DiffUtil; import android.support.v7.widget.RecyclerView; +import android.support.v7.widget.RecyclerView.ViewHolder; +import android.support.v7.widget.RecyclerView.Adapter; import android.util.Log; +import android.util.SparseArray; import android.view.LayoutInflater; import android.view.Menu; import android.view.MenuItem; @@ -24,268 +34,290 @@ import android.widget.PopupMenu; import android.widget.TextView; import android.widget.Toast; +import org.schabi.newpipe.BuildConfig; import org.schabi.newpipe.R; -import org.schabi.newpipe.download.DeleteDownloadManager; +import org.schabi.newpipe.util.NavigationHelper; import java.io.File; import java.lang.ref.WeakReference; import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import us.shandian.giga.get.DownloadManager; import us.shandian.giga.get.DownloadMission; +import us.shandian.giga.get.FinishedMission; +import us.shandian.giga.service.DownloadManager; import us.shandian.giga.service.DownloadManagerService; +import us.shandian.giga.ui.common.Deleter; import us.shandian.giga.ui.common.ProgressDrawable; import us.shandian.giga.util.Utility; import static android.content.Intent.FLAG_GRANT_PREFIX_URI_PERMISSION; import static android.content.Intent.FLAG_GRANT_READ_URI_PERMISSION; +import static us.shandian.giga.get.DownloadMission.ERROR_CONNECT_HOST; +import static us.shandian.giga.get.DownloadMission.ERROR_FILE_CREATION; +import static us.shandian.giga.get.DownloadMission.ERROR_HTTP_NO_CONTENT; +import static us.shandian.giga.get.DownloadMission.ERROR_HTTP_UNSUPPORTED_RANGE; +import static us.shandian.giga.get.DownloadMission.ERROR_NOTHING; +import static us.shandian.giga.get.DownloadMission.ERROR_PATH_CREATION; +import static us.shandian.giga.get.DownloadMission.ERROR_PERMISSION_DENIED; +import static us.shandian.giga.get.DownloadMission.ERROR_POSTPROCESSING_FAILED; +import static us.shandian.giga.get.DownloadMission.ERROR_SSL_EXCEPTION; +import static us.shandian.giga.get.DownloadMission.ERROR_UNKNOWN_EXCEPTION; +import static us.shandian.giga.get.DownloadMission.ERROR_UNKNOWN_HOST; -public class MissionAdapter extends RecyclerView.Adapter { - private static final Map ALGORITHMS = new HashMap<>(); +public class MissionAdapter extends Adapter { + private static final SparseArray ALGORITHMS = new SparseArray<>(); private static final String TAG = "MissionAdapter"; + private static final String UNDEFINED_SPEED = "--.-%"; static { ALGORITHMS.put(R.id.md5, "MD5"); ALGORITHMS.put(R.id.sha1, "SHA1"); } - private Activity mContext; + private Context mContext; private LayoutInflater mInflater; private DownloadManager mDownloadManager; - private DeleteDownloadManager mDeleteDownloadManager; - private List mItemList; - private DownloadManagerService.DMBinder mBinder; + private Deleter mDeleter; private int mLayout; + private DownloadManager.MissionIterator mIterator; + private ArrayList mPendingDownloadsItems = new ArrayList<>(); + private Handler mHandler; + private MenuItem mClear; + private View mEmptyMessage; - public MissionAdapter(Activity context, DownloadManagerService.DMBinder binder, DownloadManager downloadManager, DeleteDownloadManager deleteDownloadManager, boolean isLinear) { + public MissionAdapter(Context context, DownloadManager downloadManager, MenuItem clearButton, View emptyMessage) { mContext = context; mDownloadManager = downloadManager; - mDeleteDownloadManager = deleteDownloadManager; - mBinder = binder; + mDeleter = null; mInflater = (LayoutInflater) mContext.getSystemService(Context.LAYOUT_INFLATER_SERVICE); - mLayout = isLinear ? R.layout.mission_item_linear : R.layout.mission_item; + mLayout = R.layout.mission_item; - mItemList = new ArrayList<>(); - updateItemList(); + mHandler = new Handler(Looper.myLooper()) { + @Override + public void handleMessage(Message msg) { + switch (msg.what) { + case DownloadManagerService.MESSAGE_PROGRESS: + case DownloadManagerService.MESSAGE_ERROR: + case DownloadManagerService.MESSAGE_FINISHED: + onServiceMessage(msg); + break; + } + } + }; + + mClear = clearButton; + mEmptyMessage = emptyMessage; + + mIterator = downloadManager.getIterator(); + + checkEmptyMessageVisibility(); } - public void updateItemList() { - mItemList.clear(); - - for (int i = 0; i < mDownloadManager.getCount(); i++) { - DownloadMission mission = mDownloadManager.getMission(i); - if (!mDeleteDownloadManager.contains(mission)) { - mItemList.add(mDownloadManager.getMission(i)); - } + @Override + @NonNull + public ViewHolder onCreateViewHolder(@NonNull ViewGroup parent, int viewType) { + switch (viewType) { + case DownloadManager.SPECIAL_PENDING: + case DownloadManager.SPECIAL_FINISHED: + return new ViewHolderHeader(mInflater.inflate(R.layout.missions_header, parent, false)); } + + return new ViewHolderItem(mInflater.inflate(mLayout, parent, false)); } @Override - public MissionAdapter.ViewHolder onCreateViewHolder(ViewGroup parent, int viewType) { - final ViewHolder h = new ViewHolder(mInflater.inflate(mLayout, parent, false)); + public void onViewRecycled(@NonNull ViewHolder view) { + super.onViewRecycled(view); - h.menu.setOnClickListener(new View.OnClickListener() { - @Override - public void onClick(View v) { - buildPopup(h); - } - }); + if (view instanceof ViewHolderHeader) return; + ViewHolderItem h = (ViewHolderItem) view; - h.itemView.setOnClickListener(new View.OnClickListener() { - @Override - public void onClick(View v) { - if(h.mission.finished) viewFile(h); - } - }); + if (h.item.mission instanceof DownloadMission) { + mPendingDownloadsItems.remove(h); + if (mPendingDownloadsItems.size() < 1) setAutoRefresh(false); + } - return h; - } - - @Override - public void onViewRecycled(MissionAdapter.ViewHolder h) { - super.onViewRecycled(h); - h.mission.removeListener(h.observer); - h.mission = null; - h.observer = null; - h.progress = null; - h.position = -1; + h.popupMenu.dismiss(); + h.item = null; h.lastTimeStamp = -1; h.lastDone = -1; - h.colorId = 0; + h.lastCurrent = -1; + h.state = 0; } @Override - public void onBindViewHolder(MissionAdapter.ViewHolder h, int pos) { - DownloadMission ms = mItemList.get(pos); - h.mission = ms; - h.position = pos; + @SuppressLint("SetTextI18n") + public void onBindViewHolder(@NonNull ViewHolder view, @SuppressLint("RecyclerView") int pos) { + DownloadManager.MissionItem item = mIterator.getItem(pos); - Utility.FileType type = Utility.getFileType(ms.name); + if (view instanceof ViewHolderHeader) { + if (item.special == DownloadManager.SPECIAL_NOTHING) return; + int str; + if (item.special == DownloadManager.SPECIAL_PENDING) { + str = R.string.missions_header_pending; + } else { + str = R.string.missions_header_finished; + setClearButtonVisibility(true); + } + + ((ViewHolderHeader) view).header.setText(str); + return; + } + + ViewHolderItem h = (ViewHolderItem) view; + h.item = item; + + Utility.FileType type = Utility.getFileType(item.mission.kind, item.mission.name); h.icon.setImageResource(Utility.getIconForFileType(type)); - h.name.setText(ms.name); - h.size.setText(Utility.formatBytes(ms.length)); + h.name.setText(item.mission.name); - h.progress = new ProgressDrawable(mContext, Utility.getBackgroundForFileType(type), Utility.getForegroundForFileType(type)); - ViewCompat.setBackground(h.bkg, h.progress); + h.progress.setColors(Utility.getBackgroundForFileType(mContext, type), Utility.getForegroundForFileType(mContext, type)); - h.observer = new MissionObserver(this, h); - ms.addListener(h.observer); + if (h.item.mission instanceof DownloadMission) { + DownloadMission mission = (DownloadMission) item.mission; + String length = Utility.formatBytes(mission.getLength()); + if (mission.running && !mission.postprocessingRunning) length += " --.- kB/s"; - updateProgress(h); + h.size.setText(length); + h.pause.setTitle(mission.unknownLength ? R.string.stop : R.string.pause); + h.lastCurrent = mission.current; + updateProgress(h); + mPendingDownloadsItems.add(h); + } else { + h.progress.setMarquee(false); + h.status.setText("100%"); + h.progress.setProgress(1f); + h.size.setText(Utility.formatBytes(item.mission.length)); + } } @Override public int getItemCount() { - return mItemList.size(); + return mIterator.getOldListSize(); } @Override - public long getItemId(int position) { - return position; + public int getItemViewType(int position) { + return mIterator.getSpecialAtItem(position); } - private void updateProgress(ViewHolder h) { - updateProgress(h, false); - } - - private void updateProgress(ViewHolder h, boolean finished) { - if (h.mission == null) return; + @SuppressLint("DefaultLocale") + private void updateProgress(ViewHolderItem h) { + if (h == null || h.item == null || h.item.mission instanceof FinishedMission) return; long now = System.currentTimeMillis(); + DownloadMission mission = (DownloadMission) h.item.mission; - if (h.lastTimeStamp == -1) { + if (h.lastCurrent != mission.current) { + h.lastCurrent = mission.current; h.lastTimeStamp = now; - } - - if (h.lastDone == -1) { - h.lastDone = h.mission.done; + h.lastDone = 0; + } else { + if (h.lastTimeStamp == -1) h.lastTimeStamp = now; + if (h.lastDone == -1) h.lastDone = mission.done; } long deltaTime = now - h.lastTimeStamp; - long deltaDone = h.mission.done - h.lastDone; + long deltaDone = mission.done - h.lastDone; + boolean hasError = mission.errCode != ERROR_NOTHING; - if (deltaTime == 0 || deltaTime > 1000 || finished) { - if (h.mission.errCode > 0) { - h.status.setText(R.string.msg_error); - } else { - float progress = (float) h.mission.done / h.mission.length; - h.status.setText(String.format(Locale.US, "%.2f%%", progress * 100)); - h.progress.setProgress(progress); + // on error hide marquee or show if condition (mission.done < 1 || mission.unknownLength) is true + h.progress.setMarquee(!hasError && (mission.done < 1 || mission.unknownLength)); + + float progress; + if (mission.unknownLength) { + progress = Float.NaN; + h.progress.setProgress(0f); + } else { + progress = (float) ((double) mission.done / mission.length); + if (mission.urls.length > 1 && mission.current < mission.urls.length) { + progress = (progress / mission.urls.length) + ((float) mission.current / mission.urls.length); } } - if (deltaTime > 1000 && deltaDone > 0) { - float speed = (float) deltaDone / deltaTime; - String speedStr = Utility.formatSpeed(speed * 1000); - String sizeStr = Utility.formatBytes(h.mission.length); + if (hasError) { + if (Float.isNaN(progress) || Float.isInfinite(progress)) + h.progress.setProgress(1f); + h.status.setText(R.string.msg_error); + } else if (Float.isNaN(progress) || Float.isInfinite(progress)) { + h.status.setText(UNDEFINED_SPEED); + } else { + h.status.setText(String.format("%.2f%%", progress * 100)); + h.progress.setProgress(progress); + } - h.size.setText(sizeStr + " " + speedStr); + long length = mission.getLength(); + + int state; + if (mission.errCode == ERROR_POSTPROCESSING_FAILED) { + state = 0; + } else if (!mission.running) { + state = mission.enqueued ? 1 : 2; + } else if (mission.postprocessingRunning) { + state = 3; + } else { + state = 0; + } + + if (state != 0) { + // update state without download speed + if (h.state != state) { + String statusStr; + h.state = state; + + switch (state) { + case 1: + statusStr = mContext.getString(R.string.queued); + break; + case 2: + statusStr = mContext.getString(R.string.paused); + break; + case 3: + statusStr = mContext.getString(R.string.post_processing); + break; + default: + statusStr = "?"; + break; + } + + h.size.setText(Utility.formatBytes(length).concat(" (").concat(statusStr).concat(")")); + } else if (deltaDone > 0) { + h.lastTimeStamp = now; + h.lastDone = mission.done; + } + + return; + } + + if (deltaDone > 0 && deltaTime > 0) { + float speed = (deltaDone * 1000f) / deltaTime; + + String speedStr = Utility.formatSpeed(speed); + String sizeStr = Utility.formatBytes(length); + + h.size.setText(sizeStr.concat(" ").concat(speedStr)); h.lastTimeStamp = now; - h.lastDone = h.mission.done; + h.lastDone = mission.done; } } + private boolean viewWithFileProvider(@NonNull File file) { + if (!file.exists()) return true; - private void buildPopup(final ViewHolder h) { - PopupMenu popup = new PopupMenu(mContext, h.menu); - popup.inflate(R.menu.mission); + String ext = Utility.getFileExt(file.getName()); + if (ext == null) return false; - Menu menu = popup.getMenu(); - MenuItem start = menu.findItem(R.id.start); - MenuItem pause = menu.findItem(R.id.pause); - MenuItem delete = menu.findItem(R.id.delete); - MenuItem checksum = menu.findItem(R.id.checksum); + String mimeType = MimeTypeMap.getSingleton().getMimeTypeFromExtension(ext.substring(1)); + Log.v(TAG, "Mime: " + mimeType + " package: " + BuildConfig.APPLICATION_ID + ".provider"); - // Set to false first - start.setVisible(false); - pause.setVisible(false); - delete.setVisible(false); - checksum.setVisible(false); + Uri uri = FileProvider.getUriForFile(mContext, BuildConfig.APPLICATION_ID + ".provider", file); - if (!h.mission.finished) { - if (!h.mission.running) { - if (h.mission.errCode == -1) { - start.setVisible(true); - } - - delete.setVisible(true); - } else { - pause.setVisible(true); - } - } else { - delete.setVisible(true); - checksum.setVisible(true); - } - - popup.setOnMenuItemClickListener(new PopupMenu.OnMenuItemClickListener() { - @Override - public boolean onMenuItemClick(MenuItem item) { - int id = item.getItemId(); - switch (id) { - case R.id.start: - mDownloadManager.resumeMission(h.position); - mBinder.onMissionAdded(mItemList.get(h.position)); - return true; - case R.id.pause: - mDownloadManager.pauseMission(h.position); - mBinder.onMissionRemoved(mItemList.get(h.position)); - h.lastTimeStamp = -1; - h.lastDone = -1; - return true; - case R.id.delete: - mDeleteDownloadManager.add(h.mission); - updateItemList(); - notifyDataSetChanged(); - return true; - case R.id.md5: - case R.id.sha1: - DownloadMission mission = mItemList.get(h.position); - new ChecksumTask(mContext).execute(mission.location + "/" + mission.name, ALGORITHMS.get(id)); - return true; - default: - return false; - } - } - }); - - popup.show(); - } - - private boolean viewFile(ViewHolder h) { - File f = new File(h.mission.location, h.mission.name); - String ext = Utility.getFileExt(h.mission.name); - - Log.d(TAG, "Viewing file: " + f.getAbsolutePath() + " ext: " + ext); - - if (ext == null) { - Log.w(TAG, "Can't view file because it has no extension: " + - h.mission.name); - return true; - } - - String mime = MimeTypeMap.getSingleton().getMimeTypeFromExtension(ext.substring(1)); - Log.v(TAG, "Mime: " + mime + " package: " + mContext.getApplicationContext().getPackageName() + ".provider"); - if (f.exists()) { - viewFileWithFileProvider(f, mime); - } else { - Log.w(TAG, "File doesn't exist"); - } - return false; - } - - private void viewFileWithFileProvider(File file, String mimetype) { - String ourPackage = mContext.getApplicationContext().getPackageName(); - Uri uri = FileProvider.getUriForFile(mContext, ourPackage + ".provider", file); Intent intent = new Intent(); intent.setAction(Intent.ACTION_VIEW); - intent.setDataAndType(uri, mimetype); + intent.setDataAndType(uri, mimeType); intent.addFlags(FLAG_GRANT_READ_URI_PERMISSION); if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { intent.addFlags(FLAG_GRANT_PREFIX_URI_PERMISSION); @@ -298,75 +330,396 @@ public class MissionAdapter extends RecyclerView.Adapter= 100 && mission.errCode < 600) { + str = new StringBuilder(8); + str.append("HTTP "); + str.append(mission.errCode); + } else if (mission.errObject == null) { + str.append("(not_decelerated_error_code)"); + } + break; } - @Override - public void onProgressUpdate(DownloadMission downloadMission, long done, long total) { - mAdapter.updateProgress(mHolder); + if (mission.errObject != null) { + str.append("\n\n"); + str.append(mission.errObject.toString()); } - @Override - public void onFinish(DownloadMission downloadMission) { - //mAdapter.mManager.deleteMission(mHolder.position); - // TODO Notification - //mAdapter.notifyDataSetChanged(); - if (mHolder.mission != null) { - mHolder.size.setText(Utility.formatBytes(mHolder.mission.length)); - mAdapter.updateProgress(mHolder, true); + AlertDialog.Builder builder = new AlertDialog.Builder(mContext); + builder.setTitle(mission.name) + .setMessage(str) + .setNegativeButton(android.R.string.ok, (dialog, which) -> dialog.cancel()) + .create() + .show(); + } + + public void clearFinishedDownloads() { + mDownloadManager.forgetFinishedDownloads(); + applyChanges(); + setClearButtonVisibility(false); + } + + private boolean handlePopupItem(@NonNull ViewHolderItem h, @NonNull MenuItem option) { + int id = option.getItemId(); + DownloadMission mission = h.item.mission instanceof DownloadMission ? (DownloadMission) h.item.mission : null; + + if (mission != null) { + switch (id) { + case R.id.start: + h.state = -1; + h.size.setText(Utility.formatBytes(mission.getLength())); + mDownloadManager.resumeMission(mission); + return true; + case R.id.pause: + h.state = -1; + mDownloadManager.pauseMission(mission); + updateProgress(h); + h.lastTimeStamp = -1; + h.lastDone = -1; + return true; + case R.id.error_message_view: + showError(mission); + return true; + case R.id.queue: + h.queue.setChecked(!h.queue.isChecked()); + mission.enqueued = h.queue.isChecked(); + updateProgress(h); + return true; } } - @Override - public void onError(DownloadMission downloadMission, int errCode) { - mAdapter.updateProgress(mHolder); + switch (id) { + case R.id.open: + return viewWithFileProvider(h.item.mission.getDownloadedFile()); + case R.id.delete: + if (mDeleter == null) { + mDownloadManager.deleteMission(h.item.mission); + } else { + mDeleter.append(h.item.mission); + } + applyChanges(); + return true; + case R.id.md5: + case R.id.sha1: + new ChecksumTask(mContext).execute(h.item.mission.getDownloadedFile().getAbsolutePath(), ALGORITHMS.get(id)); + return true; + case R.id.source: + /*Intent intent = new Intent(Intent.ACTION_VIEW, Uri.parse(h.item.mission.source)); + mContext.startActivity(intent);*/ + try { + Intent intent = NavigationHelper.getIntentByLink(mContext, h.item.mission.source); + intent.addFlags(Intent.FLAG_ACTIVITY_PREVIOUS_IS_TOP); + mContext.startActivity(intent); + } catch (Exception e) { + Log.w(TAG, "Selected item has a invalid source", e); + } + return true; + default: + return false; } - } - private static class ChecksumTask extends AsyncTask { - ProgressDialog prog; - final WeakReference weakReference; + public void applyChanges() { + mIterator.start(); + DiffUtil.calculateDiff(mIterator, true).dispatchUpdatesTo(this); + mIterator.end(); - ChecksumTask(@NonNull Activity activity) { - weakReference = new WeakReference<>(activity); + checkEmptyMessageVisibility(); + + if (mIterator.getOldListSize() > 0) { + int lastItemType = mIterator.getSpecialAtItem(mIterator.getOldListSize() - 1); + setClearButtonVisibility(lastItemType == DownloadManager.SPECIAL_FINISHED); + } + } + + public void forceUpdate() { + mIterator.start(); + mIterator.end(); + + for (ViewHolderItem item : mPendingDownloadsItems) { + item.lastTimeStamp = -1; + } + + notifyDataSetChanged(); + } + + public void setLinear(boolean isLinear) { + mLayout = isLinear ? R.layout.mission_item_linear : R.layout.mission_item; + } + + public void setClearButton(MenuItem clearButton) { + if (mClear == null) { + int lastItemType = mIterator.getSpecialAtItem(mIterator.getOldListSize() - 1); + clearButton.setVisible(lastItemType == DownloadManager.SPECIAL_FINISHED); + } + mClear = clearButton; + } + + private void setClearButtonVisibility(boolean flag) { + mClear.setVisible(flag); + } + + private void checkEmptyMessageVisibility() { + int flag = mIterator.getOldListSize() > 0 ? View.GONE : View.VISIBLE; + if (mEmptyMessage.getVisibility() != flag) mEmptyMessage.setVisibility(flag); + } + + + public void deleterDispose(Bundle bundle) { + if (mDeleter != null) mDeleter.dispose(bundle); + } + + public void deleterLoad(Bundle bundle, View view) { + if (mDeleter == null) + mDeleter = new Deleter(bundle, view, mContext, this, mDownloadManager, mIterator, mHandler); + } + + public void deleterResume() { + if (mDeleter != null) mDeleter.resume(); + } + + + private boolean mUpdaterRunning = false; + private final Runnable rUpdater = this::updater; + + public void onPaused() { + setAutoRefresh(false); + } + + private void setAutoRefresh(boolean enabled) { + if (enabled && !mUpdaterRunning) { + mUpdaterRunning = true; + updater(); + } else if (!enabled && mUpdaterRunning) { + mUpdaterRunning = false; + mHandler.removeCallbacks(rUpdater); + } + } + + private void updater() { + if (!mUpdaterRunning) return; + + boolean running = false; + for (ViewHolderItem h : mPendingDownloadsItems) { + // check if the mission is running first + if (!((DownloadMission) h.item.mission).running) continue; + + updateProgress(h); + running = true; + } + + if (running) { + mHandler.postDelayed(rUpdater, 1000); + } else { + mUpdaterRunning = false; + } + } + + + class ViewHolderItem extends RecyclerView.ViewHolder { + DownloadManager.MissionItem item; + + TextView status; + ImageView icon; + TextView name; + TextView size; + ProgressDrawable progress; + + PopupMenu popupMenu; + MenuItem start; + MenuItem pause; + MenuItem open; + MenuItem queue; + MenuItem showError; + MenuItem delete; + MenuItem source; + MenuItem checksum; + + long lastTimeStamp = -1; + long lastDone = -1; + int lastCurrent = -1; + int state = 0; + + ViewHolderItem(View view) { + super(view); + + progress = new ProgressDrawable(); + ViewCompat.setBackground(itemView.findViewById(R.id.item_bkg), progress); + + status = itemView.findViewById(R.id.item_status); + name = itemView.findViewById(R.id.item_name); + icon = itemView.findViewById(R.id.item_icon); + size = itemView.findViewById(R.id.item_size); + + name.setSelected(true); + + ImageView button = itemView.findViewById(R.id.item_more); + popupMenu = buildPopup(button); + button.setOnClickListener(v -> showPopupMenu()); + + Menu menu = popupMenu.getMenu(); + start = menu.findItem(R.id.start); + pause = menu.findItem(R.id.pause); + open = menu.findItem(R.id.open); + queue = menu.findItem(R.id.queue); + showError = menu.findItem(R.id.error_message_view); + delete = menu.findItem(R.id.delete); + source = menu.findItem(R.id.source); + checksum = menu.findItem(R.id.checksum); + + itemView.setOnClickListener((v) -> { + if (item.mission instanceof FinishedMission) + viewWithFileProvider(item.mission.getDownloadedFile()); + }); + } + + private void showPopupMenu() { + start.setVisible(false); + pause.setVisible(false); + open.setVisible(false); + queue.setVisible(false); + showError.setVisible(false); + delete.setVisible(false); + source.setVisible(false); + checksum.setVisible(false); + + DownloadMission mission = item.mission instanceof DownloadMission ? (DownloadMission) item.mission : null; + + if (mission != null) { + if (!mission.postprocessingRunning) { + if (mission.running) { + pause.setVisible(true); + } else { + if (mission.errCode != ERROR_NOTHING) { + showError.setVisible(true); + } + + queue.setChecked(mission.enqueued); + + delete.setVisible(true); + start.setVisible(mission.errCode != ERROR_POSTPROCESSING_FAILED); + queue.setVisible(mission.errCode != ERROR_POSTPROCESSING_FAILED); + } + } + } else { + open.setVisible(true); + delete.setVisible(true); + checksum.setVisible(true); + } + + if (item.mission.source != null && !item.mission.source.isEmpty()) { + source.setVisible(true); + } + + popupMenu.show(); + } + + private PopupMenu buildPopup(final View button) { + PopupMenu popup = new PopupMenu(mContext, button); + popup.inflate(R.menu.mission); + popup.setOnMenuItemClickListener(option -> handlePopupItem(this, option)); + + return popup; + } + } + + class ViewHolderHeader extends RecyclerView.ViewHolder { + TextView header; + + ViewHolderHeader(View view) { + super(view); + header = itemView.findViewById(R.id.item_name); + } + } + + + static class ChecksumTask extends AsyncTask { + ProgressDialog progressDialog; + WeakReference weakReference; + + ChecksumTask(@NonNull Context context) { + weakReference = new WeakReference<>((Activity) context); } @Override @@ -376,10 +729,10 @@ public class MissionAdapter extends RecyclerView.Adapter items; + private boolean running = true; + + private Context mContext; + private MissionAdapter mAdapter; + private DownloadManager mDownloadManager; + private MissionIterator mIterator; + private Handler mHandler; + private View mView; + + private final Runnable rShow; + private final Runnable rNext; + private final Runnable rCommit; + + public Deleter(Bundle b, View v, Context c, MissionAdapter a, DownloadManager d, MissionIterator i, Handler h) { + mView = v; + mContext = c; + mAdapter = a; + mDownloadManager = d; + mIterator = i; + mHandler = h; + + // use variables to know the reference of the lambdas + rShow = this::show; + rNext = this::next; + rCommit = this::commit; + + items = new ArrayList<>(2); + + if (b != null) { + String[] names = b.getStringArray(BUNDLE_NAMES); + String[] locations = b.getStringArray(BUNDLE_LOCATIONS); + + if (names == null || locations == null) return; + if (names.length < 1 || locations.length < 1) return; + if (names.length != locations.length) return; + + items.ensureCapacity(names.length); + + for (int j = 0; j < locations.length; j++) { + Mission mission = mDownloadManager.getAnyMission(locations[j], names[j]); + if (mission == null) continue; + + items.add(mission); + mIterator.hide(mission); + } + + if (items.size() > 0) resume(); + } + } + + public void append(Mission item) { + mIterator.hide(item); + items.add(0, item); + + show(); + } + + private void forget() { + mIterator.unHide(items.remove(0)); + mAdapter.applyChanges(); + + show(); + } + + private void show() { + if (items.size() < 1) return; + + pause(); + running = true; + + mHandler.postDelayed(rNext, DELAY); + } + + private void next() { + if (items.size() < 1) return; + + String msg = mContext.getString(R.string.file_deleted).concat(":\n").concat(items.get(0).name); + + snackbar = Snackbar.make(mView, msg, Snackbar.LENGTH_INDEFINITE); + snackbar.setAction(R.string.undo, s -> forget()); + snackbar.setActionTextColor(Color.YELLOW); + snackbar.show(); + + mHandler.postDelayed(rCommit, TIMEOUT); + } + + private void commit() { + if (items.size() < 1) return; + + while (items.size() > 0) { + Mission mission = items.remove(0); + if (mission.deleted) continue; + + mIterator.unHide(mission); + mDownloadManager.deleteMission(mission); + + if (mission instanceof FinishedMission) { + mContext.sendBroadcast(new Intent(Intent.ACTION_MEDIA_SCANNER_SCAN_FILE, Uri.fromFile(mission.getDownloadedFile()))); + } + break; + } + + if (items.size() < 1) { + pause(); + return; + } + + show(); + } + + private void pause() { + running = false; + mHandler.removeCallbacks(rNext); + mHandler.removeCallbacks(rShow); + mHandler.removeCallbacks(rCommit); + if (snackbar != null) snackbar.dismiss(); + } + + public void resume() { + if (running) return; + mHandler.postDelayed(rShow, DELAY_RESUME); + } + + public void dispose(Bundle bundle) { + if (items.size() < 1) return; + + pause(); + + if (bundle == null) { + for (Mission mission : items) mDownloadManager.deleteMission(mission); + items = null; + return; + } + + String[] names = new String[items.size()]; + String[] locations = new String[items.size()]; + + for (int i = 0; i < items.size(); i++) { + Mission mission = items.get(i); + names[i] = mission.name; + locations[i] = mission.location; + } + + bundle.putStringArray(BUNDLE_NAMES, names); + bundle.putStringArray(BUNDLE_LOCATIONS, locations); + } +} diff --git a/app/src/main/java/us/shandian/giga/ui/common/ProgressDrawable.java b/app/src/main/java/us/shandian/giga/ui/common/ProgressDrawable.java index 955ce4c65..33eba22eb 100644 --- a/app/src/main/java/us/shandian/giga/ui/common/ProgressDrawable.java +++ b/app/src/main/java/us/shandian/giga/ui/common/ProgressDrawable.java @@ -1,25 +1,36 @@ package us.shandian.giga.ui.common; -import android.content.Context; import android.graphics.Canvas; import android.graphics.ColorFilter; import android.graphics.Paint; +import android.graphics.Path; import android.graphics.PixelFormat; +import android.graphics.Rect; import android.graphics.drawable.Drawable; -import android.support.annotation.ColorRes; +import android.os.Handler; +import android.os.Looper; +import android.support.annotation.ColorInt; import android.support.annotation.NonNull; -import android.support.v4.content.ContextCompat; public class ProgressDrawable extends Drawable { - private float mProgress; - private final int mBackgroundColor; - private final int mForegroundColor; + private static final int MARQUEE_INTERVAL = 150; - public ProgressDrawable(Context context, @ColorRes int background, @ColorRes int foreground) { - this(ContextCompat.getColor(context, background), ContextCompat.getColor(context, foreground)); + private float mProgress; + private int mBackgroundColor, mForegroundColor; + private Handler mMarqueeHandler; + private float mMarqueeProgress; + private Path mMarqueeLine; + private int mMarqueeSize; + private long mMarqueeNext; + + public ProgressDrawable() { + mMarqueeLine = null;// marquee disabled + mMarqueeProgress = 0f; + mMarqueeSize = 0; + mMarqueeNext = 0; } - public ProgressDrawable(int background, int foreground) { + public void setColors(@ColorInt int background, @ColorInt int foreground) { mBackgroundColor = background; mForegroundColor = foreground; } @@ -29,10 +40,20 @@ public class ProgressDrawable extends Drawable { invalidateSelf(); } + public void setMarquee(boolean marquee) { + if (marquee == (mMarqueeLine != null)) { + return; + } + mMarqueeLine = marquee ? new Path() : null; + mMarqueeHandler = marquee ? new Handler(Looper.getMainLooper()) : null; + mMarqueeSize = 0; + mMarqueeNext = 0; + } + @Override public void draw(@NonNull Canvas canvas) { - int width = canvas.getWidth(); - int height = canvas.getHeight(); + int width = getBounds().width(); + int height = getBounds().height(); Paint paint = new Paint(); @@ -40,6 +61,42 @@ public class ProgressDrawable extends Drawable { canvas.drawRect(0, 0, width, height, paint); paint.setColor(mForegroundColor); + + if (mMarqueeLine != null) { + if (mMarqueeSize < 1) setupMarquee(width, height); + + int size = mMarqueeSize; + Paint paint2 = new Paint(); + paint2.setColor(mForegroundColor); + paint2.setStrokeWidth(size); + paint2.setStyle(Paint.Style.STROKE); + + size *= 2; + + if (mMarqueeProgress >= size) { + mMarqueeProgress = 1; + } else { + mMarqueeProgress++; + } + + // render marquee + width += size * 2; + Path marquee = new Path(); + for (float i = -size; i < width; i += size) { + marquee.addPath(mMarqueeLine, i + mMarqueeProgress, 0); + } + marquee.close(); + + canvas.drawPath(marquee, paint2);// draw marquee + + if (System.currentTimeMillis() >= mMarqueeNext) { + // program next update + mMarqueeNext = System.currentTimeMillis() + MARQUEE_INTERVAL; + mMarqueeHandler.postDelayed(this::invalidateSelf, MARQUEE_INTERVAL); + } + return; + } + canvas.drawRect(0, 0, (int) (mProgress * width), height, paint); } @@ -58,4 +115,17 @@ public class ProgressDrawable extends Drawable { return PixelFormat.OPAQUE; } + @Override + public void onBoundsChange(Rect rect) { + if (mMarqueeLine != null) setupMarquee(rect.width(), rect.height()); + } + + private void setupMarquee(int width, int height) { + mMarqueeSize = (int) ((width * 10f) / 100f);// the size is 10% of the width + + mMarqueeLine.rewind(); + mMarqueeLine.moveTo(-mMarqueeSize, -mMarqueeSize); + mMarqueeLine.lineTo(-mMarqueeSize * 4, height + mMarqueeSize); + mMarqueeLine.close(); + } } diff --git a/app/src/main/java/us/shandian/giga/ui/fragment/AllMissionsFragment.java b/app/src/main/java/us/shandian/giga/ui/fragment/AllMissionsFragment.java deleted file mode 100644 index ec8d7fc22..000000000 --- a/app/src/main/java/us/shandian/giga/ui/fragment/AllMissionsFragment.java +++ /dev/null @@ -1,12 +0,0 @@ -package us.shandian.giga.ui.fragment; - -import us.shandian.giga.get.DownloadManager; -import us.shandian.giga.service.DownloadManagerService; - -public class AllMissionsFragment extends MissionsFragment { - - @Override - protected DownloadManager setupDownloadManager(DownloadManagerService.DMBinder binder) { - return binder.getDownloadManager(); - } -} diff --git a/app/src/main/java/us/shandian/giga/ui/fragment/MissionsFragment.java b/app/src/main/java/us/shandian/giga/ui/fragment/MissionsFragment.java index 5241415b2..aa9c497f1 100644 --- a/app/src/main/java/us/shandian/giga/ui/fragment/MissionsFragment.java +++ b/app/src/main/java/us/shandian/giga/ui/fragment/MissionsFragment.java @@ -10,52 +10,58 @@ import android.content.SharedPreferences; import android.os.Bundle; import android.os.IBinder; import android.preference.PreferenceManager; -import android.support.annotation.NonNull; -import android.support.annotation.Nullable; import android.support.v7.widget.GridLayoutManager; import android.support.v7.widget.LinearLayoutManager; import android.support.v7.widget.RecyclerView; import android.view.LayoutInflater; import android.view.Menu; -import android.view.MenuInflater; import android.view.MenuItem; import android.view.View; import android.view.ViewGroup; import org.schabi.newpipe.R; -import org.schabi.newpipe.download.DeleteDownloadManager; -import io.reactivex.disposables.Disposable; -import us.shandian.giga.get.DownloadManager; +import us.shandian.giga.service.DownloadManager; import us.shandian.giga.service.DownloadManagerService; +import us.shandian.giga.service.DownloadManagerService.DMBinder; import us.shandian.giga.ui.adapter.MissionAdapter; -public abstract class MissionsFragment extends Fragment { - private DownloadManager mDownloadManager; - private DownloadManagerService.DMBinder mBinder; +public class MissionsFragment extends Fragment { + + private static final int SPAN_SIZE = 2; private SharedPreferences mPrefs; private boolean mLinear; private MenuItem mSwitch; + private MenuItem mClear = null; private RecyclerView mList; + private View mEmpty; private MissionAdapter mAdapter; private GridLayoutManager mGridManager; private LinearLayoutManager mLinearManager; private Context mActivity; - private DeleteDownloadManager mDeleteDownloadManager; - private Disposable mDeleteDisposable; - private final ServiceConnection mConnection = new ServiceConnection() { + private DMBinder mBinder; + private Bundle mBundle; + private boolean mForceUpdate; + + private ServiceConnection mConnection = new ServiceConnection() { @Override public void onServiceConnected(ComponentName name, IBinder binder) { mBinder = (DownloadManagerService.DMBinder) binder; - mDownloadManager = setupDownloadManager(mBinder); - if (mDeleteDownloadManager != null) { - mDeleteDownloadManager.setDownloadManager(mDownloadManager); - updateList(); - } + mBinder.clearDownloadNotifications(); + + mAdapter = new MissionAdapter(mActivity, mBinder.getDownloadManager(), mClear, mEmpty); + mAdapter.deleterLoad(mBundle, getView()); + + mBundle = null; + + mBinder.addMissionEventListener(mAdapter.getMessenger()); + mBinder.enableNotifications(false); + + updateList(); } @Override @@ -66,14 +72,6 @@ public abstract class MissionsFragment extends Fragment { }; - public void setDeleteManager(@NonNull DeleteDownloadManager deleteDownloadManager) { - mDeleteDownloadManager = deleteDownloadManager; - if (mDownloadManager != null) { - mDeleteDownloadManager.setDownloadManager(mDownloadManager); - updateList(); - } - } - @Override public View onCreateView(LayoutInflater inflater, ViewGroup container, Bundle savedInstanceState) { View v = inflater.inflate(R.layout.missions, container, false); @@ -81,18 +79,32 @@ public abstract class MissionsFragment extends Fragment { mPrefs = PreferenceManager.getDefaultSharedPreferences(getActivity()); mLinear = mPrefs.getBoolean("linear", false); + mActivity = getActivity(); + mBundle = savedInstanceState; + // Bind the service - Intent i = new Intent(); - i.setClass(getActivity(), DownloadManagerService.class); - getActivity().bindService(i, mConnection, Context.BIND_AUTO_CREATE); + mActivity.bindService(new Intent(mActivity, DownloadManagerService.class), mConnection, Context.BIND_AUTO_CREATE); // Views + mEmpty = v.findViewById(R.id.list_empty_view); mList = v.findViewById(R.id.mission_recycler); // Init - mGridManager = new GridLayoutManager(getActivity(), 2); + mGridManager = new GridLayoutManager(getActivity(), SPAN_SIZE); + mGridManager.setSpanSizeLookup(new GridLayoutManager.SpanSizeLookup() { + @Override + public int getSpanSize(int position) { + switch (mAdapter.getItemViewType(position)) { + case DownloadManager.SPECIAL_PENDING: + case DownloadManager.SPECIAL_FINISHED: + return SPAN_SIZE; + default: + return 1; + } + } + }); + mLinearManager = new LinearLayoutManager(getActivity()); - mList.setLayoutManager(mGridManager); setHasOptionsMenu(true); @@ -123,31 +135,26 @@ public abstract class MissionsFragment extends Fragment { mActivity = activity; } - @Override - public void onViewCreated(View view, @Nullable Bundle savedInstanceState) { - super.onViewCreated(view, savedInstanceState); - if (mDeleteDownloadManager != null) { - mDeleteDisposable = mDeleteDownloadManager.getUndoObservable().subscribe(mission -> { - if (mAdapter != null) { - mAdapter.updateItemList(); - mAdapter.notifyDataSetChanged(); - } - }); - } - } @Override - public void onDestroyView() { - super.onDestroyView(); - getActivity().unbindService(mConnection); - if (mDeleteDisposable != null) { - mDeleteDisposable.dispose(); - } + public void onDestroy() { + super.onDestroy(); + if (mBinder == null || mAdapter == null) return; + + mBinder.removeMissionEventListener(mAdapter.getMessenger()); + mBinder.enableNotifications(true); + mActivity.unbindService(mConnection); + mAdapter.deleterDispose(null); + + mBinder = null; + mAdapter = null; } @Override public void onPrepareOptionsMenu(Menu menu) { mSwitch = menu.findItem(R.id.switch_mode); + mClear = menu.findItem(R.id.clear_list); + if (mAdapter != null) mAdapter.setClearButton(mClear); super.onPrepareOptionsMenu(menu); } @@ -155,35 +162,71 @@ public abstract class MissionsFragment extends Fragment { public boolean onOptionsItemSelected(MenuItem item) { switch (item.getItemId()) { case R.id.switch_mode: - mLinear = !mLinear; - updateList(); - return true; - default: - return super.onOptionsItemSelected(item); - } - } - - public void notifyChange() { - mAdapter.notifyDataSetChanged(); + mLinear = !mLinear; + updateList(); + return true; + case R.id.clear_list: + mAdapter.clearFinishedDownloads(); + return true; + default: + return super.onOptionsItemSelected(item); + } } private void updateList() { - mAdapter = new MissionAdapter((Activity) mActivity, mBinder, mDownloadManager, mDeleteDownloadManager, mLinear); - if (mLinear) { mList.setLayoutManager(mLinearManager); } else { mList.setLayoutManager(mGridManager); } + // destroy all created views in the recycler + mList.setAdapter(null); + mAdapter.notifyDataSetChanged(); + + // re-attach the adapter in grid/lineal mode + mAdapter.setLinear(mLinear); mList.setAdapter(mAdapter); if (mSwitch != null) { mSwitch.setIcon(mLinear ? R.drawable.grid : R.drawable.list); + mSwitch.setTitle(mLinear ? R.string.grid : R.string.list); + mPrefs.edit().putBoolean("linear", mLinear).apply(); } - - mPrefs.edit().putBoolean("linear", mLinear).apply(); } - protected abstract DownloadManager setupDownloadManager(DownloadManagerService.DMBinder binder); + @Override + public void onSaveInstanceState(Bundle outState) { + super.onSaveInstanceState(outState); + + if (mAdapter != null) { + mAdapter.deleterDispose(outState); + mForceUpdate = true; + mBinder.removeMissionEventListener(mAdapter.getMessenger()); + } + } + + @Override + public void onResume() { + super.onResume(); + + if (mAdapter != null) { + mAdapter.deleterResume(); + + if (mForceUpdate) { + mForceUpdate = false; + mAdapter.forceUpdate(); + } + + mBinder.addMissionEventListener(mAdapter.getMessenger()); + } + if (mBinder != null) mBinder.enableNotifications(false); + } + + @Override + public void onPause() { + super.onPause(); + if (mAdapter != null) mAdapter.onPaused(); + if (mBinder != null) mBinder.enableNotifications(true); + } } diff --git a/app/src/main/java/us/shandian/giga/util/Utility.java b/app/src/main/java/us/shandian/giga/util/Utility.java index 163ac2b14..e5149cf9b 100644 --- a/app/src/main/java/us/shandian/giga/util/Utility.java +++ b/app/src/main/java/us/shandian/giga/util/Utility.java @@ -3,15 +3,18 @@ package us.shandian.giga.util; import android.content.ClipData; import android.content.ClipboardManager; import android.content.Context; -import android.support.annotation.ColorRes; +import android.os.Build; +import android.support.annotation.ColorInt; import android.support.annotation.DrawableRes; import android.support.annotation.NonNull; import android.support.annotation.Nullable; +import android.support.v4.content.ContextCompat; import android.widget.Toast; import org.schabi.newpipe.R; import java.io.BufferedOutputStream; +import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileOutputStream; @@ -19,14 +22,17 @@ import java.io.IOException; import java.io.ObjectInputStream; import java.io.ObjectOutputStream; import java.io.Serializable; +import java.net.HttpURLConnection; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; +import java.util.Locale; public class Utility { public enum FileType { VIDEO, MUSIC, + SUBTITLE, UNKNOWN } @@ -34,11 +40,11 @@ public class Utility { if (bytes < 1024) { return String.format("%d B", bytes); } else if (bytes < 1024 * 1024) { - return String.format("%.2f kB", (float) bytes / 1024); + return String.format("%.2f kB", bytes / 1024d); } else if (bytes < 1024 * 1024 * 1024) { - return String.format("%.2f MB", (float) bytes / 1024 / 1024); + return String.format("%.2f MB", bytes / 1024d / 1024d); } else { - return String.format("%.2f GB", (float) bytes / 1024 / 1024 / 1024); + return String.format("%.2f GB", bytes / 1024d / 1024d / 1024d); } } @@ -54,41 +60,32 @@ public class Utility { } } - public static void writeToFile(@NonNull String fileName, @NonNull Serializable serializable) { - ObjectOutputStream objectOutputStream = null; + public static void writeToFile(@NonNull File file, @NonNull Serializable serializable) { - try { - objectOutputStream = new ObjectOutputStream(new BufferedOutputStream(new FileOutputStream(fileName))); + try (ObjectOutputStream objectOutputStream = new ObjectOutputStream(new BufferedOutputStream(new FileOutputStream(file)))) { objectOutputStream.writeObject(serializable); } catch (Exception e) { //nothing to do - } finally { - if(objectOutputStream != null) { - try { - objectOutputStream.close(); - } catch (Exception e) { - //nothing to do - } - } } + //nothing to do } @Nullable @SuppressWarnings("unchecked") - public static T readFromFile(String file) { - T object = null; + public static T readFromFile(File file) { + T object; ObjectInputStream objectInputStream = null; try { objectInputStream = new ObjectInputStream(new FileInputStream(file)); object = (T) objectInputStream.readObject(); } catch (Exception e) { - //nothing to do + object = null; } - if(objectInputStream != null){ + if (objectInputStream != null) { try { - objectInputStream .close(); + objectInputStream.close(); } catch (Exception e) { //nothing to do } @@ -119,39 +116,68 @@ public class Utility { } } - public static FileType getFileType(String file) { - if (file.endsWith(".mp3") || file.endsWith(".wav") || file.endsWith(".flac") || file.endsWith(".m4a")) { + public static FileType getFileType(char kind, String file) { + switch (kind) { + case 'v': + return FileType.VIDEO; + case 'a': + return FileType.MUSIC; + case 's': + return FileType.SUBTITLE; + //default '?': + } + + if (file.endsWith(".srt") || file.endsWith(".vtt") || file.endsWith(".ssa")) { + return FileType.SUBTITLE; + } else if (file.endsWith(".mp3") || file.endsWith(".wav") || file.endsWith(".flac") || file.endsWith(".m4a") || file.endsWith(".opus")) { return FileType.MUSIC; } else if (file.endsWith(".mp4") || file.endsWith(".mpeg") || file.endsWith(".rm") || file.endsWith(".rmvb") || file.endsWith(".flv") || file.endsWith(".webp") || file.endsWith(".webm")) { return FileType.VIDEO; - } else { - return FileType.UNKNOWN; } + + return FileType.UNKNOWN; } - @ColorRes - public static int getBackgroundForFileType(FileType type) { + @ColorInt + public static int getBackgroundForFileType(Context ctx, FileType type) { + int colorRes; switch (type) { case MUSIC: - return R.color.audio_left_to_load_color; + colorRes = R.color.audio_left_to_load_color; + break; case VIDEO: - return R.color.video_left_to_load_color; + colorRes = R.color.video_left_to_load_color; + break; + case SUBTITLE: + colorRes = R.color.subtitle_left_to_load_color; + break; default: - return R.color.gray; + colorRes = R.color.gray; } + + return ContextCompat.getColor(ctx, colorRes); } - @ColorRes - public static int getForegroundForFileType(FileType type) { + @ColorInt + public static int getForegroundForFileType(Context ctx, FileType type) { + int colorRes; switch (type) { case MUSIC: - return R.color.audio_already_load_color; + colorRes = R.color.audio_already_load_color; + break; case VIDEO: - return R.color.video_already_load_color; + colorRes = R.color.video_already_load_color; + break; + case SUBTITLE: + colorRes = R.color.subtitle_already_load_color; + break; default: - return R.color.gray; + colorRes = R.color.gray; + break; } + + return ContextCompat.getColor(ctx, colorRes); } @DrawableRes @@ -161,6 +187,8 @@ public class Utility { return R.drawable.music; case VIDEO: return R.drawable.video; + case SUBTITLE: + return R.drawable.subtitle; default: return R.drawable.video; } @@ -168,12 +196,18 @@ public class Utility { public static void copyToClipboard(Context context, String str) { ClipboardManager cm = (ClipboardManager) context.getSystemService(Context.CLIPBOARD_SERVICE); + + if (cm == null) { + Toast.makeText(context, R.string.permission_denied, Toast.LENGTH_LONG).show(); + return; + } + cm.setPrimaryClip(ClipData.newPlainText("text", str)); Toast.makeText(context, R.string.msg_copied, Toast.LENGTH_SHORT).show(); } public static String checksum(String path, String algorithm) { - MessageDigest md = null; + MessageDigest md; try { md = MessageDigest.getInstance(algorithm); @@ -181,7 +215,7 @@ public class Utility { throw new RuntimeException(e); } - FileInputStream i = null; + FileInputStream i; try { i = new FileInputStream(path); @@ -190,14 +224,14 @@ public class Utility { } byte[] buf = new byte[1024]; - int len = 0; + int len; try { while ((len = i.read(buf)) != -1) { md.update(buf, 0, len); } - } catch (IOException ignored) { - + } catch (IOException e) { + // nothing to do } byte[] digest = md.digest(); @@ -211,4 +245,31 @@ public class Utility { return sb.toString(); } + + @SuppressWarnings("ResultOfMethodCallIgnored") + public static boolean mkdir(File path, boolean allDirs) { + if (path.exists()) return true; + + if (allDirs) + path.mkdirs(); + else + path.mkdir(); + + return path.exists(); + } + + public static long getContentLength(HttpURLConnection connection) { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) { + return connection.getContentLengthLong(); + } + + try { + long length = Long.parseLong(connection.getHeaderField("Content-Length")); + if (length >= 0) return length; + } catch (Exception err) { + // nothing to do + } + + return -1; + } } diff --git a/app/src/main/res/drawable-hdpi/grid.png b/app/src/main/res/drawable-hdpi/grid.png index 254f1d300..26fa36c07 100644 Binary files a/app/src/main/res/drawable-hdpi/grid.png and b/app/src/main/res/drawable-hdpi/grid.png differ diff --git a/app/src/main/res/drawable-hdpi/list.png b/app/src/main/res/drawable-hdpi/list.png index 0b3f54c20..16da863e2 100644 Binary files a/app/src/main/res/drawable-hdpi/list.png and b/app/src/main/res/drawable-hdpi/list.png differ diff --git a/app/src/main/res/drawable-xhdpi/subtitle.png b/app/src/main/res/drawable-xhdpi/subtitle.png new file mode 100644 index 000000000..7f535288e Binary files /dev/null and b/app/src/main/res/drawable-xhdpi/subtitle.png differ diff --git a/app/src/main/res/layout/download_dialog.xml b/app/src/main/res/layout/download_dialog.xml index 2cdfee553..985ce03f5 100644 --- a/app/src/main/res/layout/download_dialog.xml +++ b/app/src/main/res/layout/download_dialog.xml @@ -53,6 +53,12 @@ android:layout_width="wrap_content" android:layout_height="wrap_content" android:text="@string/audio"/> + + @@ -61,7 +61,9 @@ android:layout_below="@id/item_icon" android:padding="6dp" android:singleLine="true" - android:ellipsize="end" + android:ellipsize="marquee" + android:marqueeRepeatLimit="marquee_forever" + android:scrollHorizontally="true" android:text="XXX.xx" android:textSize="16sp" android:textStyle="bold" diff --git a/app/src/main/res/layout/mission_item_linear.xml b/app/src/main/res/layout/mission_item_linear.xml index 0133d0c3f..7fff76235 100644 --- a/app/src/main/res/layout/mission_item_linear.xml +++ b/app/src/main/res/layout/mission_item_linear.xml @@ -56,6 +56,7 @@ android:layout_toRightOf="@id/item_size" android:padding="6dp" android:singleLine="true" + android:textStyle="bold" android:text="0%" android:textColor="@color/white" android:textSize="12sp" /> diff --git a/app/src/main/res/layout/missions.xml b/app/src/main/res/layout/missions.xml index 3abfecd8e..431ad4769 100644 --- a/app/src/main/res/layout/missions.xml +++ b/app/src/main/res/layout/missions.xml @@ -1,9 +1,14 @@ - + + + + + + + + + + + + diff --git a/app/src/main/res/menu/download_menu.xml b/app/src/main/res/menu/download_menu.xml index e71eaf152..e79367135 100644 --- a/app/src/main/res/menu/download_menu.xml +++ b/app/src/main/res/menu/download_menu.xml @@ -2,10 +2,18 @@ + + - - \ No newline at end of file + android:title="@string/clear_finished_download"/> + diff --git a/app/src/main/res/menu/mission.xml b/app/src/main/res/menu/mission.xml index c1688ac5b..961e12fd0 100644 --- a/app/src/main/res/menu/mission.xml +++ b/app/src/main/res/menu/mission.xml @@ -1,33 +1,48 @@ + - - - + - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + diff --git a/app/src/main/res/values-es/strings.xml b/app/src/main/res/values-es/strings.xml index 9c06e228b..3ad9c6874 100644 --- a/app/src/main/res/values-es/strings.xml +++ b/app/src/main/res/values-es/strings.xml @@ -500,4 +500,55 @@ abrir en modo popup Usuarios Listas de reproducción Pistas + + Finalizadas + En cola + + pausado + en cola + post-procesado + + Encolar + + Acción denegada por el sistema + + Archivo borrado + + + Descarga fallida + Descarga finalizada + %s descargas finalizadas + + + Generar nombre único + Sobrescribir + Ya existe un archivo descargado con este nombre + Hay una descarga en curso con este nombre + + Mostrar como grilla + Mostrar como lista + Limpiar descargas finalizadas + Tienes %s descargas pendientes, ve a Descargas para continuarlas + Detener + Intentos maximos + Cantidad máxima de intentos antes de cancelar la descarga + Pausar al cambiar a datos moviles + No todas las descargas se pueden suspender, en esos casos, se reiniciaran + + + + Mostrar error + Codigo + No se puede crear la carpeta de destino + No se puede crear el archivo + Permiso denegado por el sistema + Fallo la conexión segura + No se puede encontrar el servidor + No se puede conectar con el servidor + El servidor no devolvio datos + El servidor no acepta descargas multi-hilos, intente de nuevo con @string/msg_threads = 1 + Rango solicitado no satisfactorio + No encontrado + Fallo el post-procesado + diff --git a/app/src/main/res/values/colors.xml b/app/src/main/res/values/colors.xml index 515f1d46f..5741d1b4f 100644 --- a/app/src/main/res/values/colors.xml +++ b/app/src/main/res/values/colors.xml @@ -63,6 +63,8 @@ #000000 #CD5656 #BC211D + #008ea4 + #005a71 #FFFFFF diff --git a/app/src/main/res/values/settings_keys.xml b/app/src/main/res/values/settings_keys.xml index b1ff9471d..300217c09 100644 --- a/app/src/main/res/values/settings_keys.xml +++ b/app/src/main/res/values/settings_keys.xml @@ -17,6 +17,7 @@ use_external_video_player use_external_audio_player autoplay_through_intent + use_oldplayer volume_gesture_control brightness_gesture_control @@ -173,6 +174,24 @@ @string/charset_most_special_characters_value + + downloads_max_retry + 3 + + @string/minimize_on_exit_none_description + 1 + 2 + 3 + 4 + 5 + 7 + 10 + 15 + + + cross_network_downloads + + default_download_threads preferred_open_action_key diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 1a2265e79..ec366ebed 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -143,6 +143,7 @@ Resizing Best resolution Undo + File deleted Play All Always Just Once @@ -523,4 +524,51 @@ Grid Auto Switch View + + + Finished + In queue + + paused + queued + post-processing + + Queue + + Action denied by the system + + + Download failed + Download finished + %s downloads finished + + + Generate unique name + Overwrite + A downloaded file with this name already exists + There is a download in progress with this name + + + Show error + Code + The file can not be created + The destination folder can not be created + Permission denied by the system + Secure connection failed + Can not found the server + Can not connect to the server + The server does not send data + The server does not accept multi-threaded downloads, retry with @string/msg_threads = 1 + Requested Range Not Satisfiable + Not found + Post-processing failed + + Clear finished downloads + You have %s pending downloads, goto Downloads to continue + Stop + Maximum retry + Maximum number of attempts before canceling the download + Pause on switching to mobile data + Not all downloads can be suspended, in those cases, will be restarted + diff --git a/app/src/main/res/xml/download_settings.xml b/app/src/main/res/xml/download_settings.xml index 0a8768e9e..e5d2031fe 100644 --- a/app/src/main/res/xml/download_settings.xml +++ b/app/src/main/res/xml/download_settings.xml @@ -29,4 +29,18 @@ android:summary="@string/settings_file_replacement_character_summary" android:title="@string/settings_file_replacement_character_title"/> + + + + diff --git a/app/src/test/java/us/shandian/giga/get/DownloadManagerImplTest.java b/app/src/test/java/us/shandian/giga/get/DownloadManagerImplTest.java deleted file mode 100644 index c755ba2e9..000000000 --- a/app/src/test/java/us/shandian/giga/get/DownloadManagerImplTest.java +++ /dev/null @@ -1,186 +0,0 @@ -package us.shandian.giga.get; - -import org.junit.Ignore; -import org.junit.Test; - -import java.io.File; -import java.io.IOException; -import java.io.RandomAccessFile; -import java.util.ArrayList; - -import us.shandian.giga.get.DownloadDataSource; -import us.shandian.giga.get.DownloadManagerImpl; -import us.shandian.giga.get.DownloadMission; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertSame; -import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -/** - * Test for {@link DownloadManagerImpl} - * - * TODO: test loading from .giga files, startMission and improve tests - */ -public class DownloadManagerImplTest { - - private DownloadManagerImpl downloadManager; - private DownloadDataSource downloadDataSource; - private ArrayList missions; - - @org.junit.Before - public void setUp() throws Exception { - downloadDataSource = mock(DownloadDataSource.class); - missions = new ArrayList<>(); - for(int i = 0; i < 50; ++i){ - missions.add(generateFinishedDownloadMission()); - } - when(downloadDataSource.loadMissions()).thenReturn(new ArrayList<>(missions)); - downloadManager = new DownloadManagerImpl(new ArrayList<>(), downloadDataSource); - } - - @Test(expected = NullPointerException.class) - public void testConstructorWithNullAsDownloadDataSource() { - new DownloadManagerImpl(new ArrayList<>(), null); - } - - - private static DownloadMission generateFinishedDownloadMission() throws IOException { - File file = File.createTempFile("newpipetest", ".mp4"); - file.deleteOnExit(); - RandomAccessFile randomAccessFile = new RandomAccessFile(file, "rw"); - randomAccessFile.setLength(1000); - randomAccessFile.close(); - DownloadMission downloadMission = new DownloadMission(file.getName(), - "http://google.com/?q=how+to+google", file.getParent()); - downloadMission.blocks = 1000; - downloadMission.done = 1000; - downloadMission.finished = true; - return spy(downloadMission); - } - - private static void assertMissionEquals(String message, DownloadMission expected, DownloadMission actual) { - if(expected == actual) return; - assertEquals(message + ": Name", expected.name, actual.name); - assertEquals(message + ": Location", expected.location, actual.location); - assertEquals(message + ": Url", expected.url, actual.url); - } - - @Test - public void testThatMissionsAreLoaded() throws IOException { - ArrayList missions = new ArrayList<>(); - long millis = System.currentTimeMillis(); - for(int i = 0; i < 50; ++i){ - DownloadMission mission = generateFinishedDownloadMission(); - mission.timestamp = millis - i; // reverse order by timestamp - missions.add(mission); - } - - downloadDataSource = mock(DownloadDataSource.class); - when(downloadDataSource.loadMissions()).thenReturn(new ArrayList<>(missions)); - downloadManager = new DownloadManagerImpl(new ArrayList<>(), downloadDataSource); - verify(downloadDataSource, times(1)).loadMissions(); - - assertEquals(50, downloadManager.getCount()); - - for(int i = 0; i < 50; ++i) { - assertMissionEquals("mission " + i, missions.get(50 - 1 - i), downloadManager.getMission(i)); - } - } - - @Ignore - @Test - public void startMission() throws Exception { - DownloadMission mission = missions.get(0); - mission = spy(mission); - missions.set(0, mission); - String url = "https://github.com/favicon.ico"; - // create a temp file and delete it so we have a temp directory - File tempFile = File.createTempFile("favicon",".ico"); - String name = tempFile.getName(); - String location = tempFile.getParent(); - assertTrue(tempFile.delete()); - int id = downloadManager.startMission(url, location, name, true, 10); - } - - @Test - public void resumeMission() { - DownloadMission mission = missions.get(0); - mission.running = true; - verify(mission, never()).start(); - downloadManager.resumeMission(0); - verify(mission, never()).start(); - mission.running = false; - downloadManager.resumeMission(0); - verify(mission, times(1)).start(); - } - - @Test - public void pauseMission() { - DownloadMission mission = missions.get(0); - mission.running = false; - downloadManager.pauseMission(0); - verify(mission, never()).pause(); - mission.running = true; - downloadManager.pauseMission(0); - verify(mission, times(1)).pause(); - } - - @Test - public void deleteMission() { - DownloadMission mission = missions.get(0); - assertEquals(mission, downloadManager.getMission(0)); - downloadManager.deleteMission(0); - verify(mission, times(1)).delete(); - assertNotEquals(mission, downloadManager.getMission(0)); - assertEquals(49, downloadManager.getCount()); - } - - @Test(expected = RuntimeException.class) - public void getMissionWithNegativeIndex() { - downloadManager.getMission(-1); - } - - @Test - public void getMission() { - assertSame(missions.get(0), downloadManager.getMission(0)); - assertSame(missions.get(1), downloadManager.getMission(1)); - } - - @Test - public void sortByTimestamp() { - ArrayList downloadMissions = new ArrayList<>(); - DownloadMission mission = new DownloadMission(); - mission.timestamp = 0; - - DownloadMission mission1 = new DownloadMission(); - mission1.timestamp = Integer.MAX_VALUE + 1L; - - DownloadMission mission2 = new DownloadMission(); - mission2.timestamp = 2L * Integer.MAX_VALUE ; - - DownloadMission mission3 = new DownloadMission(); - mission3.timestamp = 2L * Integer.MAX_VALUE + 5L; - - - downloadMissions.add(mission3); - downloadMissions.add(mission1); - downloadMissions.add(mission2); - downloadMissions.add(mission); - - - DownloadManagerImpl.sortByTimestamp(downloadMissions); - - assertEquals(mission, downloadMissions.get(0)); - assertEquals(mission1, downloadMissions.get(1)); - assertEquals(mission2, downloadMissions.get(2)); - assertEquals(mission3, downloadMissions.get(3)); - } - -} \ No newline at end of file