Merge pull request #12014 from hi-ogawa/fix-video-playback

Fix video playback

7 files changed, 241 insertions, 105 deletions

diff --git a/modules/webm/config.py b/modules/webm/config.py
index ef5daca05c..fb920482f5 100644
--- a/modules/webm/config.py
+++ b/modules/webm/config.py
@@ -1,7 +1,6 @@
 
 def can_build(platform):
-#    return True
-    return False
+    return True
 
 
 def configure(env):
diff --git a/modules/webm/libvpx/SCsub b/modules/webm/libvpx/SCsub
index fd8d762a5e..73ba17d184 100644
--- a/modules/webm/libvpx/SCsub
+++ b/modules/webm/libvpx/SCsub
@@ -298,7 +298,7 @@ if webm_cpu_x86:
 
     if not yasm_found:
         webm_cpu_x86 = False
-        print "YASM is necessary for WebM SIMD optimizations."
+        print("YASM is necessary for WebM SIMD optimizations.")
 
 webm_simd_optimizations = False
 
@@ -345,7 +345,7 @@ if webm_cpu_arm:
     webm_simd_optimizations = True
 
 if webm_simd_optimizations == False:
-    print "WebM SIMD optimizations are disabled. Check if your CPU architecture, CPU bits or platform are supported!"
+    print("WebM SIMD optimizations are disabled. Check if your CPU architecture, CPU bits or platform are supported!")
 
 
 env_libvpx.add_source_files(env.modules_sources, libvpx_sources)
diff --git a/modules/webm/register_types.cpp b/modules/webm/register_types.cpp
index 892d1b8420..669c9997f1 100644
--- a/modules/webm/register_types.cpp
+++ b/modules/webm/register_types.cpp
@@ -28,19 +28,18 @@
 /* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.                */
 /*************************************************************************/
 #include "register_types.h"
-
+#include "resource_importer_webm.h"
 #include "video_stream_webm.h"
 
-static ResourceFormatLoaderVideoStreamWebm *webm_stream_loader = NULL;
-
 void register_webm_types() {
 
-	webm_stream_loader = memnew(ResourceFormatLoaderVideoStreamWebm);
-	ResourceLoader::add_resource_format_loader(webm_stream_loader);
+#ifdef TOOLS_ENABLED
+	Ref<ResourceImporterWebm> webm_import;
+	webm_import.instance();
+	ResourceFormatImporter::get_singleton()->add_importer(webm_import);
+#endif
 	ClassDB::register_class<VideoStreamWebm>();
 }
 
 void unregister_webm_types() {
-
-	memdelete(webm_stream_loader);
 }
diff --git a/modules/webm/resource_importer_webm.cpp b/modules/webm/resource_importer_webm.cpp
new file mode 100644
index 0000000000..5db3d4df2e
--- /dev/null
+++ b/modules/webm/resource_importer_webm.cpp
@@ -0,0 +1,95 @@
+/*************************************************************************/
+/*  resource_importer_webm.cpp                                           */
+/*************************************************************************/
+/*                       This file is part of:                           */
+/*                           GODOT ENGINE                                */
+/*                    http://www.godotengine.org                         */
+/*************************************************************************/
+/* Copyright (c) 2007-2017 Juan Linietsky, Ariel Manzur.                 */
+/* Copyright (c) 2014-2017 Godot Engine contributors (cf. AUTHORS.md)    */
+/*                                                                       */
+/* Permission is hereby granted, free of charge, to any person obtaining */
+/* a copy of this software and associated documentation files (the       */
+/* "Software"), to deal in the Software without restriction, including   */
+/* without limitation the rights to use, copy, modify, merge, publish,   */
+/* distribute, sublicense, and/or sell copies of the Software, and to    */
+/* permit persons to whom the Software is furnished to do so, subject to */
+/* the following conditions:                                             */
+/*                                                                       */
+/* The above copyright notice and this permission notice shall be        */
+/* included in all copies or substantial portions of the Software.       */
+/*                                                                       */
+/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,       */
+/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF    */
+/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/
+/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY  */
+/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,  */
+/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE     */
+/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.                */
+/*************************************************************************/
+#include "resource_importer_webm.h"
+
+#include "io/resource_saver.h"
+#include "os/file_access.h"
+#include "scene/resources/texture.h"
+#include "video_stream_webm.h"
+
+String ResourceImporterWebm::get_importer_name() const {
+
+	return "Webm";
+}
+
+String ResourceImporterWebm::get_visible_name() const {
+
+	return "Webm";
+}
+void ResourceImporterWebm::get_recognized_extensions(List<String> *p_extensions) const {
+
+	p_extensions->push_back("webm");
+}
+
+String ResourceImporterWebm::get_save_extension() const {
+	return "webmstr";
+}
+
+String ResourceImporterWebm::get_resource_type() const {
+
+	return "VideoStreamWebm";
+}
+
+bool ResourceImporterWebm::get_option_visibility(const String &p_option, const Map<StringName, Variant> &p_options) const {
+
+	return true;
+}
+
+int ResourceImporterWebm::get_preset_count() const {
+	return 0;
+}
+String ResourceImporterWebm::get_preset_name(int p_idx) const {
+
+	return String();
+}
+
+void ResourceImporterWebm::get_import_options(List<ImportOption> *r_options, int p_preset) const {
+
+	r_options->push_back(ImportOption(PropertyInfo(Variant::BOOL, "loop"), true));
+}
+
+Error ResourceImporterWebm::import(const String &p_source_file, const String &p_save_path, const Map<StringName, Variant> &p_options, List<String> *r_platform_variants, List<String> *r_gen_files) {
+
+	FileAccess *f = FileAccess::open(p_source_file, FileAccess::READ);
+	if (!f) {
+		ERR_FAIL_COND_V(!f, ERR_CANT_OPEN);
+	}
+	memdelete(f);
+
+	VideoStreamWebm *stream = memnew(VideoStreamWebm);
+	stream->set_file(p_source_file);
+
+	Ref<VideoStreamWebm> webm_stream = Ref<VideoStreamWebm>(stream);
+
+	return ResourceSaver::save(p_save_path + ".webmstr", webm_stream);
+}
+
+ResourceImporterWebm::ResourceImporterWebm() {
+}
diff --git a/modules/webm/resource_importer_webm.h b/modules/webm/resource_importer_webm.h
new file mode 100644
index 0000000000..4cedd1598d
--- /dev/null
+++ b/modules/webm/resource_importer_webm.h
@@ -0,0 +1,55 @@
+/*************************************************************************/
+/*  resource_importer_webm.h                                             */
+/*************************************************************************/
+/*                       This file is part of:                           */
+/*                           GODOT ENGINE                                */
+/*                    http://www.godotengine.org                         */
+/*************************************************************************/
+/* Copyright (c) 2007-2017 Juan Linietsky, Ariel Manzur.                 */
+/* Copyright (c) 2014-2017 Godot Engine contributors (cf. AUTHORS.md)    */
+/*                                                                       */
+/* Permission is hereby granted, free of charge, to any person obtaining */
+/* a copy of this software and associated documentation files (the       */
+/* "Software"), to deal in the Software without restriction, including   */
+/* without limitation the rights to use, copy, modify, merge, publish,   */
+/* distribute, sublicense, and/or sell copies of the Software, and to    */
+/* permit persons to whom the Software is furnished to do so, subject to */
+/* the following conditions:                                             */
+/*                                                                       */
+/* The above copyright notice and this permission notice shall be        */
+/* included in all copies or substantial portions of the Software.       */
+/*                                                                       */
+/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,       */
+/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF    */
+/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/
+/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY  */
+/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,  */
+/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE     */
+/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.                */
+/*************************************************************************/
+#ifndef RESOURCEIMPORTERWEBM_H
+#define RESOURCEIMPORTERWEBM_H
+
+#include "io/resource_import.h"
+
+class ResourceImporterWebm : public ResourceImporter {
+	GDCLASS(ResourceImporterWebm, ResourceImporter)
+public:
+	virtual String get_importer_name() const;
+	virtual String get_visible_name() const;
+	virtual void get_recognized_extensions(List<String> *p_extensions) const;
+	virtual String get_save_extension() const;
+	virtual String get_resource_type() const;
+
+	virtual int get_preset_count() const;
+	virtual String get_preset_name(int p_idx) const;
+
+	virtual void get_import_options(List<ImportOption> *r_options, int p_preset = 0) const;
+	virtual bool get_option_visibility(const String &p_option, const Map<StringName, Variant> &p_options) const;
+
+	virtual Error import(const String &p_source_file, const String &p_save_path, const Map<StringName, Variant> &p_options, List<String> *r_platform_variants, List<String> *r_gen_files = NULL);
+
+	ResourceImporterWebm();
+};
+
+#endif // RESOURCEIMPORTERWEBM_H
diff --git a/modules/webm/video_stream_webm.cpp b/modules/webm/video_stream_webm.cpp
index 2ec6b27471..0fc9df5b58 100644
--- a/modules/webm/video_stream_webm.cpp
+++ b/modules/webm/video_stream_webm.cpp
@@ -35,10 +35,13 @@
 #include "mkvparser/mkvparser.h"
 
 #include "os/file_access.h"
+#include "os/os.h"
 #include "project_settings.h"
 
 #include "thirdparty/misc/yuv2rgb.h"
 
+#include "servers/audio_server.h"
+
 #include <string.h>
 
 class MkvReader : public mkvparser::IMkvReader {
@@ -47,6 +50,8 @@ public:
 	MkvReader(const String &p_file) {
 
 		file = FileAccess::open(p_file, FileAccess::READ);
+
+		ERR_EXPLAIN("Failed loading resource: '" + p_file + "';");
 		ERR_FAIL_COND(!file);
 	}
 	~MkvReader() {
@@ -113,14 +118,14 @@ bool VideoStreamPlaybackWebm::open_file(const String &p_file) {
 	webm = memnew(WebMDemuxer(new MkvReader(file_name), 0, audio_track));
 	if (webm->isOpen()) {
 
-		video = memnew(VPXDecoder(*webm, 8)); //TODO: Detect CPU threads
+		video = memnew(VPXDecoder(*webm, OS::get_singleton()->get_processor_count()));
 		if (video->isOpen()) {
 
 			audio = memnew(OpusVorbisDecoder(*webm));
 			if (audio->isOpen()) {
 
 				audio_frame = memnew(WebMFrame);
-				pcm = (int16_t *)memalloc(sizeof(int16_t) * audio->getBufferSamples() * webm->getChannels());
+				pcm = (float *)memalloc(sizeof(float) * audio->getBufferSamples() * webm->getChannels());
 			} else {
 
 				memdelete(audio);
@@ -183,7 +188,7 @@ void VideoStreamPlaybackWebm::set_paused(bool p_paused) {
 
 	paused = p_paused;
 }
-bool VideoStreamPlaybackWebm::is_paused(bool p_paused) const {
+bool VideoStreamPlaybackWebm::is_paused() const {
 
 	return paused;
 }
@@ -222,11 +227,18 @@ Ref<Texture> VideoStreamPlaybackWebm::get_texture() {
 
 	return texture;
 }
+
 void VideoStreamPlaybackWebm::update(float p_delta) {
 
 	if ((!playing || paused) || !video)
 		return;
 
+	time += p_delta;
+
+	if (time < video_pos) {
+		return;
+	}
+
 	bool audio_buffer_full = false;
 
 	if (samples_offset > -1) {
@@ -245,13 +257,15 @@ void VideoStreamPlaybackWebm::update(float p_delta) {
 	}
 
 	const bool hasAudio = (audio && mix_callback);
-	while ((hasAudio && (!audio_buffer_full || !has_enough_video_frames())) || (!hasAudio && video_frames_pos == 0)) {
+	while ((hasAudio && !audio_buffer_full && !has_enough_video_frames()) ||
+			(!hasAudio && video_frames_pos == 0)) {
 
-		if (hasAudio && !audio_buffer_full && audio_frame->isValid() && audio->getPCMS16(*audio_frame, pcm, num_decoded_samples) && num_decoded_samples > 0) {
+		if (hasAudio && !audio_buffer_full && audio_frame->isValid() &&
+				audio->getPCMF(*audio_frame, pcm, num_decoded_samples) && num_decoded_samples > 0) {
 
 			const int mixed = mix_callback(mix_udata, pcm, num_decoded_samples);
-			if (mixed != num_decoded_samples) {
 
+			if (mixed != num_decoded_samples) {
 				samples_offset = mixed;
 				audio_buffer_full = true;
 			}
@@ -273,72 +287,61 @@ void VideoStreamPlaybackWebm::update(float p_delta) {
 			++video_frames_pos;
 	};
 
-	const double video_delay = video->getFramesDelay() * video_frame_delay;
-
-	bool want_this_frame = false;
-	while (video_frames_pos > 0 && !want_this_frame) {
+	bool video_frame_done = false;
+	while (video_frames_pos > 0 && !video_frame_done) {
 
 		WebMFrame *video_frame = video_frames[0];
-		if (video_frame->time <= time + video_delay) {
 
-			if (video->decode(*video_frame)) {
+		// It seems VPXDecoder::decode has to be executed even though we might skip this frame
+		if (video->decode(*video_frame)) {
 
-				VPXDecoder::IMAGE_ERROR err;
-				VPXDecoder::Image image;
+			VPXDecoder::IMAGE_ERROR err;
+			VPXDecoder::Image image;
 
-				while ((err = video->getImage(image)) != VPXDecoder::NO_FRAME) {
+			if (should_process(*video_frame)) {
 
-					want_this_frame = (time - video_frame->time <= video_frame_delay);
+				if ((err = video->getImage(image)) != VPXDecoder::NO_FRAME) {
 
-					if (want_this_frame) {
+					if (err == VPXDecoder::NO_ERROR && image.w == webm->getWidth() && image.h == webm->getHeight()) {
 
-						if (err == VPXDecoder::NO_ERROR && image.w == webm->getWidth() && image.h == webm->getHeight()) {
+						PoolVector<uint8_t>::Write w = frame_data.write();
+						bool converted = false;
 
-							PoolVector<uint8_t>::Write w = frame_data.write();
-							bool converted = false;
+						if (image.chromaShiftW == 1 && image.chromaShiftH == 1) {
 
-							if (image.chromaShiftW == 1 && image.chromaShiftH == 1) {
+							yuv420_2_rgb8888(w.ptr(), image.planes[0], image.planes[2], image.planes[1], image.w, image.h, image.linesize[0], image.linesize[1], image.w << 2, 0);
+							// 								libyuv::I420ToARGB(image.planes[0], image.linesize[0], image.planes[2], image.linesize[2], image.planes[1], image.linesize[1], w.ptr(), image.w << 2, image.w, image.h);
+							converted = true;
+						} else if (image.chromaShiftW == 1 && image.chromaShiftH == 0) {
 
-								yuv420_2_rgb8888(w.ptr(), image.planes[0], image.planes[2], image.planes[1], image.w, image.h, image.linesize[0], image.linesize[1], image.w << 2, 0);
-								// 								libyuv::I420ToARGB(image.planes[0], image.linesize[0], image.planes[2], image.linesize[2], image.planes[1], image.linesize[1], w.ptr(), image.w << 2, image.w, image.h);
-								converted = true;
-							} else if (image.chromaShiftW == 1 && image.chromaShiftH == 0) {
+							yuv422_2_rgb8888(w.ptr(), image.planes[0], image.planes[2], image.planes[1], image.w, image.h, image.linesize[0], image.linesize[1], image.w << 2, 0);
+							// 								libyuv::I422ToARGB(image.planes[0], image.linesize[0], image.planes[2], image.linesize[2], image.planes[1], image.linesize[1], w.ptr(), image.w << 2, image.w, image.h);
+							converted = true;
+						} else if (image.chromaShiftW == 0 && image.chromaShiftH == 0) {
 
-								yuv422_2_rgb8888(w.ptr(), image.planes[0], image.planes[2], image.planes[1], image.w, image.h, image.linesize[0], image.linesize[1], image.w << 2, 0);
-								// 								libyuv::I422ToARGB(image.planes[0], image.linesize[0], image.planes[2], image.linesize[2], image.planes[1], image.linesize[1], w.ptr(), image.w << 2, image.w, image.h);
-								converted = true;
-							} else if (image.chromaShiftW == 0 && image.chromaShiftH == 0) {
+							yuv444_2_rgb8888(w.ptr(), image.planes[0], image.planes[2], image.planes[1], image.w, image.h, image.linesize[0], image.linesize[1], image.w << 2, 0);
+							// 								libyuv::I444ToARGB(image.planes[0], image.linesize[0], image.planes[2], image.linesize[2], image.planes[1], image.linesize[1], w.ptr(), image.w << 2, image.w, image.h);
+							converted = true;
+						} else if (image.chromaShiftW == 2 && image.chromaShiftH == 0) {
 
-								yuv444_2_rgb8888(w.ptr(), image.planes[0], image.planes[2], image.planes[1], image.w, image.h, image.linesize[0], image.linesize[1], image.w << 2, 0);
-								// 								libyuv::I444ToARGB(image.planes[0], image.linesize[0], image.planes[2], image.linesize[2], image.planes[1], image.linesize[1], w.ptr(), image.w << 2, image.w, image.h);
-								converted = true;
-							} else if (image.chromaShiftW == 2 && image.chromaShiftH == 0) {
-
-								// 								libyuv::I411ToARGB(image.planes[0], image.linesize[0], image.planes[2], image.linesize[2], image.planes[1], image.linesize[1], w.ptr(), image.w << 2, image.w, image.h);
-								// 								converted = true;
-							}
-
-							if (converted)
-								texture->set_data(Image(image.w, image.h, 0, Image::FORMAT_RGBA8, frame_data)); //Zero copy send to visual server
+							// 								libyuv::I411ToARGB(image.planes[0], image.linesize[0], image.planes[2], image.linesize[2], image.planes[1], image.linesize[1], w.ptr(), image.w << 2, image.w, image.h);
+							// 								converted = true;
 						}
 
-						break;
+						if (converted) {
+							Ref<Image> img = memnew(Image(image.w, image.h, 0, Image::FORMAT_RGBA8, frame_data));
+							texture->set_data(img); //Zero copy send to visual server
+							video_frame_done = true;
+						}
 					}
 				}
 			}
-
-			video_frame_delay = video_frame->time - video_pos;
-			video_pos = video_frame->time;
-
-			memmove(video_frames, video_frames + 1, (--video_frames_pos) * sizeof(void *));
-			video_frames[video_frames_pos] = video_frame;
-		} else {
-
-			break;
 		}
-	}
 
-	time += p_delta;
+		video_pos = video_frame->time;
+		memmove(video_frames, video_frames + 1, (--video_frames_pos) * sizeof(void *));
+		video_frames[video_frames_pos] = video_frame;
+	}
 
 	if (video_frames_pos == 0 && webm->isEOS())
 		stop();
@@ -372,6 +375,11 @@ inline bool VideoStreamPlaybackWebm::has_enough_video_frames() const {
 	return false;
 }
 
+bool VideoStreamPlaybackWebm::should_process(WebMFrame &video_frame) {
+	const double audio_delay = AudioServer::get_singleton()->get_output_delay();
+	return video_frame.time >= time + audio_delay + delay_compensation;
+}
+
 void VideoStreamPlaybackWebm::delete_pointers() {
 
 	if (pcm)
@@ -395,34 +403,6 @@ void VideoStreamPlaybackWebm::delete_pointers() {
 
 /**/
 
-RES ResourceFormatLoaderVideoStreamWebm::load(const String &p_path, const String &p_original_path, Error *r_error) {
-
-	Ref<VideoStreamWebm> stream = memnew(VideoStreamWebm);
-	stream->set_file(p_path);
-	if (r_error)
-		*r_error = OK;
-	return stream;
-}
-
-void ResourceFormatLoaderVideoStreamWebm::get_recognized_extensions(List<String> *p_extensions) const {
-
-	p_extensions->push_back("webm");
-}
-bool ResourceFormatLoaderVideoStreamWebm::handles_type(const String &p_type) const {
-
-	return (p_type == "VideoStream" || p_type == "VideoStreamWebm");
-}
-
-String ResourceFormatLoaderVideoStreamWebm::get_resource_type(const String &p_path) const {
-
-	const String exl = p_path.get_extension().to_lower();
-	if (exl == "webm")
-		return "VideoStreamWebm";
-	return "";
-}
-
-/**/
-
 VideoStreamWebm::VideoStreamWebm()
 	: audio_track(0) {}
 
@@ -439,6 +419,19 @@ void VideoStreamWebm::set_file(const String &p_file) {
 
 	file = p_file;
 }
+String VideoStreamWebm::get_file() {
+
+	return file;
+}
+
+void VideoStreamWebm::_bind_methods() {
+
+	ClassDB::bind_method(D_METHOD("set_file", "file"), &VideoStreamWebm::set_file);
+	ClassDB::bind_method(D_METHOD("get_file"), &VideoStreamWebm::get_file);
+
+	ADD_PROPERTY(PropertyInfo(Variant::STRING, "file", PROPERTY_HINT_NONE, "", PROPERTY_USAGE_NOEDITOR), "set_file", "get_file");
+}
+
 void VideoStreamWebm::set_audio_track(int p_track) {
 
 	audio_track = p_track;
diff --git a/modules/webm/video_stream_webm.h b/modules/webm/video_stream_webm.h
index fc0720967a..f7dd16a38f 100644
--- a/modules/webm/video_stream_webm.h
+++ b/modules/webm/video_stream_webm.h
@@ -60,7 +60,7 @@ class VideoStreamPlaybackWebm : public VideoStreamPlayback {
 	PoolVector<uint8_t> frame_data;
 	Ref<ImageTexture> texture;
 
-	int16_t *pcm;
+	float *pcm;
 
 public:
 	VideoStreamPlaybackWebm();
@@ -74,7 +74,7 @@ public:
 	virtual bool is_playing() const;
 
 	virtual void set_paused(bool p_paused);
-	virtual bool is_paused(bool p_paused) const;
+	virtual bool is_paused() const;
 
 	virtual void set_loop(bool p_enable);
 	virtual bool has_loop() const;
@@ -95,6 +95,7 @@ public:
 
 private:
 	inline bool has_enough_video_frames() const;
+	bool should_process(WebMFrame &video_frame);
 
 	void delete_pointers();
 };
@@ -103,27 +104,21 @@ private:
 
 class VideoStreamWebm : public VideoStream {
 
-	GDCLASS(VideoStreamWebm, VideoStream)
+	GDCLASS(VideoStreamWebm, VideoStream);
+	RES_BASE_EXTENSION("webmstr");
 
 	String file;
 	int audio_track;
 
+protected:
+	static void _bind_methods();
+
 public:
 	VideoStreamWebm();
 
 	virtual Ref<VideoStreamPlayback> instance_playback();
 
 	virtual void set_file(const String &p_file);
+	String get_file();
 	virtual void set_audio_track(int p_track);
 };
-
-/**/
-
-class ResourceFormatLoaderVideoStreamWebm : public ResourceFormatLoader {
-
-public:
-	virtual RES load(const String &p_path, const String &p_original_path, Error *r_error);
-	virtual void get_recognized_extensions(List<String> *p_extensions) const;
-	virtual bool handles_type(const String &p_type) const;
-	virtual String get_resource_type(const String &p_path) const;
-};