diff --git a/.gitattributes b/.gitattributes
index a6344aac8c09253b3b630fb776ae94478aa0275b..5cb38146a1aa0e19df300720209ca1757a1b9afb 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
*.zip filter=lfs diff=lfs merge=lfs -text
*.zst filter=lfs diff=lfs merge=lfs -text
*tfevents* filter=lfs diff=lfs merge=lfs -text
+figure/multitask.png filter=lfs diff=lfs merge=lfs -text
diff --git a/LICENSE b/LICENSE
new file mode 100644
index 0000000000000000000000000000000000000000..e62ec04cdeece724caeeeeaeb6ae1f6af1bb6b9a
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,674 @@
+GNU GENERAL PUBLIC LICENSE
+ Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc.
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+ Preamble
+
+ The GNU General Public License is a free, copyleft license for
+software and other kinds of works.
+
+ The licenses for most software and other practical works are designed
+to take away your freedom to share and change the works. By contrast,
+the GNU General Public License is intended to guarantee your freedom to
+share and change all versions of a program--to make sure it remains free
+software for all its users. We, the Free Software Foundation, use the
+GNU General Public License for most of our software; it applies also to
+any other work released this way by its authors. You can apply it to
+your programs, too.
+
+ When we speak of free software, we are referring to freedom, not
+price. Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+them if you wish), that you receive source code or can get it if you
+want it, that you can change the software or use pieces of it in new
+free programs, and that you know you can do these things.
+
+ To protect your rights, we need to prevent others from denying you
+these rights or asking you to surrender the rights. Therefore, you have
+certain responsibilities if you distribute copies of the software, or if
+you modify it: responsibilities to respect the freedom of others.
+
+ For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must pass on to the recipients the same
+freedoms that you received. You must make sure that they, too, receive
+or can get the source code. And you must show them these terms so they
+know their rights.
+
+ Developers that use the GNU GPL protect your rights with two steps:
+(1) assert copyright on the software, and (2) offer you this License
+giving you legal permission to copy, distribute and/or modify it.
+
+ For the developers' and authors' protection, the GPL clearly explains
+that there is no warranty for this free software. For both users' and
+authors' sake, the GPL requires that modified versions be marked as
+changed, so that their problems will not be attributed erroneously to
+authors of previous versions.
+
+ Some devices are designed to deny users access to install or run
+modified versions of the software inside them, although the manufacturer
+can do so. This is fundamentally incompatible with the aim of
+protecting users' freedom to change the software. The systematic
+pattern of such abuse occurs in the area of products for individuals to
+use, which is precisely where it is most unacceptable. Therefore, we
+have designed this version of the GPL to prohibit the practice for those
+products. If such problems arise substantially in other domains, we
+stand ready to extend this provision to those domains in future versions
+of the GPL, as needed to protect the freedom of users.
+
+ Finally, every program is threatened constantly by software patents.
+States should not allow patents to restrict development and use of
+software on general-purpose computers, but in those that do, we wish to
+avoid the special danger that patents applied to a free program could
+make it effectively proprietary. To prevent this, the GPL assures that
+patents cannot be used to render the program non-free.
+
+ The precise terms and conditions for copying, distribution and
+modification follow.
+
+ TERMS AND CONDITIONS
+
+ 0. Definitions.
+
+ "This License" refers to version 3 of the GNU General Public License.
+
+ "Copyright" also means copyright-like laws that apply to other kinds of
+works, such as semiconductor masks.
+
+ "The Program" refers to any copyrightable work licensed under this
+License. Each licensee is addressed as "you". "Licensees" and
+"recipients" may be individuals or organizations.
+
+ To "modify" a work means to copy from or adapt all or part of the work
+in a fashion requiring copyright permission, other than the making of an
+exact copy. The resulting work is called a "modified version" of the
+earlier work or a work "based on" the earlier work.
+
+ A "covered work" means either the unmodified Program or a work based
+on the Program.
+
+ To "propagate" a work means to do anything with it that, without
+permission, would make you directly or secondarily liable for
+infringement under applicable copyright law, except executing it on a
+computer or modifying a private copy. Propagation includes copying,
+distribution (with or without modification), making available to the
+public, and in some countries other activities as well.
+
+ To "convey" a work means any kind of propagation that enables other
+parties to make or receive copies. Mere interaction with a user through
+a computer network, with no transfer of a copy, is not conveying.
+
+ An interactive user interface displays "Appropriate Legal Notices"
+to the extent that it includes a convenient and prominently visible
+feature that (1) displays an appropriate copyright notice, and (2)
+tells the user that there is no warranty for the work (except to the
+extent that warranties are provided), that licensees may convey the
+work under this License, and how to view a copy of this License. If
+the interface presents a list of user commands or options, such as a
+menu, a prominent item in the list meets this criterion.
+
+ 1. Source Code.
+
+ The "source code" for a work means the preferred form of the work
+for making modifications to it. "Object code" means any non-source
+form of a work.
+
+ A "Standard Interface" means an interface that either is an official
+standard defined by a recognized standards body, or, in the case of
+interfaces specified for a particular programming language, one that
+is widely used among developers working in that language.
+
+ The "System Libraries" of an executable work include anything, other
+than the work as a whole, that (a) is included in the normal form of
+packaging a Major Component, but which is not part of that Major
+Component, and (b) serves only to enable use of the work with that
+Major Component, or to implement a Standard Interface for which an
+implementation is available to the public in source code form. A
+"Major Component", in this context, means a major essential component
+(kernel, window system, and so on) of the specific operating system
+(if any) on which the executable work runs, or a compiler used to
+produce the work, or an object code interpreter used to run it.
+
+ The "Corresponding Source" for a work in object code form means all
+the source code needed to generate, install, and (for an executable
+work) run the object code and to modify the work, including scripts to
+control those activities. However, it does not include the work's
+System Libraries, or general-purpose tools or generally available free
+programs which are used unmodified in performing those activities but
+which are not part of the work. For example, Corresponding Source
+includes interface definition files associated with source files for
+the work, and the source code for shared libraries and dynamically
+linked subprograms that the work is specifically designed to require,
+such as by intimate data communication or control flow between those
+subprograms and other parts of the work.
+
+ The Corresponding Source need not include anything that users
+can regenerate automatically from other parts of the Corresponding
+Source.
+
+ The Corresponding Source for a work in source code form is that
+same work.
+
+ 2. Basic Permissions.
+
+ All rights granted under this License are granted for the term of
+copyright on the Program, and are irrevocable provided the stated
+conditions are met. This License explicitly affirms your unlimited
+permission to run the unmodified Program. The output from running a
+covered work is covered by this License only if the output, given its
+content, constitutes a covered work. This License acknowledges your
+rights of fair use or other equivalent, as provided by copyright law.
+
+ You may make, run and propagate covered works that you do not
+convey, without conditions so long as your license otherwise remains
+in force. You may convey covered works to others for the sole purpose
+of having them make modifications exclusively for you, or provide you
+with facilities for running those works, provided that you comply with
+the terms of this License in conveying all material for which you do
+not control copyright. Those thus making or running the covered works
+for you must do so exclusively on your behalf, under your direction
+and control, on terms that prohibit them from making any copies of
+your copyrighted material outside their relationship with you.
+
+ Conveying under any other circumstances is permitted solely under
+the conditions stated below. Sublicensing is not allowed; section 10
+makes it unnecessary.
+
+ 3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+
+ No covered work shall be deemed part of an effective technological
+measure under any applicable law fulfilling obligations under article
+11 of the WIPO copyright treaty adopted on 20 December 1996, or
+similar laws prohibiting or restricting circumvention of such
+measures.
+
+ When you convey a covered work, you waive any legal power to forbid
+circumvention of technological measures to the extent such circumvention
+is effected by exercising rights under this License with respect to
+the covered work, and you disclaim any intention to limit operation or
+modification of the work as a means of enforcing, against the work's
+users, your or third parties' legal rights to forbid circumvention of
+technological measures.
+
+ 4. Conveying Verbatim Copies.
+
+ You may convey verbatim copies of the Program's source code as you
+receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice;
+keep intact all notices stating that this License and any
+non-permissive terms added in accord with section 7 apply to the code;
+keep intact all notices of the absence of any warranty; and give all
+recipients a copy of this License along with the Program.
+
+ You may charge any price or no price for each copy that you convey,
+and you may offer support or warranty protection for a fee.
+
+ 5. Conveying Modified Source Versions.
+
+ You may convey a work based on the Program, or the modifications to
+produce it from the Program, in the form of source code under the
+terms of section 4, provided that you also meet all of these conditions:
+
+ a) The work must carry prominent notices stating that you modified
+ it, and giving a relevant date.
+
+ b) The work must carry prominent notices stating that it is
+ released under this License and any conditions added under section
+ 7. This requirement modifies the requirement in section 4 to
+ "keep intact all notices".
+
+ c) You must license the entire work, as a whole, under this
+ License to anyone who comes into possession of a copy. This
+ License will therefore apply, along with any applicable section 7
+ additional terms, to the whole of the work, and all its parts,
+ regardless of how they are packaged. This License gives no
+ permission to license the work in any other way, but it does not
+ invalidate such permission if you have separately received it.
+
+ d) If the work has interactive user interfaces, each must display
+ Appropriate Legal Notices; however, if the Program has interactive
+ interfaces that do not display Appropriate Legal Notices, your
+ work need not make them do so.
+
+ A compilation of a covered work with other separate and independent
+works, which are not by their nature extensions of the covered work,
+and which are not combined with it such as to form a larger program,
+in or on a volume of a storage or distribution medium, is called an
+"aggregate" if the compilation and its resulting copyright are not
+used to limit the access or legal rights of the compilation's users
+beyond what the individual works permit. Inclusion of a covered work
+in an aggregate does not cause this License to apply to the other
+parts of the aggregate.
+
+ 6. Conveying Non-Source Forms.
+
+ You may convey a covered work in object code form under the terms
+of sections 4 and 5, provided that you also convey the
+machine-readable Corresponding Source under the terms of this License,
+in one of these ways:
+
+ a) Convey the object code in, or embodied in, a physical product
+ (including a physical distribution medium), accompanied by the
+ Corresponding Source fixed on a durable physical medium
+ customarily used for software interchange.
+
+ b) Convey the object code in, or embodied in, a physical product
+ (including a physical distribution medium), accompanied by a
+ written offer, valid for at least three years and valid for as
+ long as you offer spare parts or customer support for that product
+ model, to give anyone who possesses the object code either (1) a
+ copy of the Corresponding Source for all the software in the
+ product that is covered by this License, on a durable physical
+ medium customarily used for software interchange, for a price no
+ more than your reasonable cost of physically performing this
+ conveying of source, or (2) access to copy the
+ Corresponding Source from a network server at no charge.
+
+ c) Convey individual copies of the object code with a copy of the
+ written offer to provide the Corresponding Source. This
+ alternative is allowed only occasionally and noncommercially, and
+ only if you received the object code with such an offer, in accord
+ with subsection 6b.
+
+ d) Convey the object code by offering access from a designated
+ place (gratis or for a charge), and offer equivalent access to the
+ Corresponding Source in the same way through the same place at no
+ further charge. You need not require recipients to copy the
+ Corresponding Source along with the object code. If the place to
+ copy the object code is a network server, the Corresponding Source
+ may be on a different server (operated by you or a third party)
+ that supports equivalent copying facilities, provided you maintain
+ clear directions next to the object code saying where to find the
+ Corresponding Source. Regardless of what server hosts the
+ Corresponding Source, you remain obligated to ensure that it is
+ available for as long as needed to satisfy these requirements.
+
+ e) Convey the object code using peer-to-peer transmission, provided
+ you inform other peers where the object code and Corresponding
+ Source of the work are being offered to the general public at no
+ charge under subsection 6d.
+
+ A separable portion of the object code, whose source code is excluded
+from the Corresponding Source as a System Library, need not be
+included in conveying the object code work.
+
+ A "User Product" is either (1) a "consumer product", which means any
+tangible personal property which is normally used for personal, family,
+or household purposes, or (2) anything designed or sold for incorporation
+into a dwelling. In determining whether a product is a consumer product,
+doubtful cases shall be resolved in favor of coverage. For a particular
+product received by a particular user, "normally used" refers to a
+typical or common use of that class of product, regardless of the status
+of the particular user or of the way in which the particular user
+actually uses, or expects or is expected to use, the product. A product
+is a consumer product regardless of whether the product has substantial
+commercial, industrial or non-consumer uses, unless such uses represent
+the only significant mode of use of the product.
+
+ "Installation Information" for a User Product means any methods,
+procedures, authorization keys, or other information required to install
+and execute modified versions of a covered work in that User Product from
+a modified version of its Corresponding Source. The information must
+suffice to ensure that the continued functioning of the modified object
+code is in no case prevented or interfered with solely because
+modification has been made.
+
+ If you convey an object code work under this section in, or with, or
+specifically for use in, a User Product, and the conveying occurs as
+part of a transaction in which the right of possession and use of the
+User Product is transferred to the recipient in perpetuity or for a
+fixed term (regardless of how the transaction is characterized), the
+Corresponding Source conveyed under this section must be accompanied
+by the Installation Information. But this requirement does not apply
+if neither you nor any third party retains the ability to install
+modified object code on the User Product (for example, the work has
+been installed in ROM).
+
+ The requirement to provide Installation Information does not include a
+requirement to continue to provide support service, warranty, or updates
+for a work that has been modified or installed by the recipient, or for
+the User Product in which it has been modified or installed. Access to a
+network may be denied when the modification itself materially and
+adversely affects the operation of the network or violates the rules and
+protocols for communication across the network.
+
+ Corresponding Source conveyed, and Installation Information provided,
+in accord with this section must be in a format that is publicly
+documented (and with an implementation available to the public in
+source code form), and must require no special password or key for
+unpacking, reading or copying.
+
+ 7. Additional Terms.
+
+ "Additional permissions" are terms that supplement the terms of this
+License by making exceptions from one or more of its conditions.
+Additional permissions that are applicable to the entire Program shall
+be treated as though they were included in this License, to the extent
+that they are valid under applicable law. If additional permissions
+apply only to part of the Program, that part may be used separately
+under those permissions, but the entire Program remains governed by
+this License without regard to the additional permissions.
+
+ When you convey a copy of a covered work, you may at your option
+remove any additional permissions from that copy, or from any part of
+it. (Additional permissions may be written to require their own
+removal in certain cases when you modify the work.) You may place
+additional permissions on material, added by you to a covered work,
+for which you have or can give appropriate copyright permission.
+
+ Notwithstanding any other provision of this License, for material you
+add to a covered work, you may (if authorized by the copyright holders of
+that material) supplement the terms of this License with terms:
+
+ a) Disclaiming warranty or limiting liability differently from the
+ terms of sections 15 and 16 of this License; or
+
+ b) Requiring preservation of specified reasonable legal notices or
+ author attributions in that material or in the Appropriate Legal
+ Notices displayed by works containing it; or
+
+ c) Prohibiting misrepresentation of the origin of that material, or
+ requiring that modified versions of such material be marked in
+ reasonable ways as different from the original version; or
+
+ d) Limiting the use for publicity purposes of names of licensors or
+ authors of the material; or
+
+ e) Declining to grant rights under trademark law for use of some
+ trade names, trademarks, or service marks; or
+
+ f) Requiring indemnification of licensors and authors of that
+ material by anyone who conveys the material (or modified versions of
+ it) with contractual assumptions of liability to the recipient, for
+ any liability that these contractual assumptions directly impose on
+ those licensors and authors.
+
+ All other non-permissive additional terms are considered "further
+restrictions" within the meaning of section 10. If the Program as you
+received it, or any part of it, contains a notice stating that it is
+governed by this License along with a term that is a further
+restriction, you may remove that term. If a license document contains
+a further restriction but permits relicensing or conveying under this
+License, you may add to a covered work material governed by the terms
+of that license document, provided that the further restriction does
+not survive such relicensing or conveying.
+
+ If you add terms to a covered work in accord with this section, you
+must place, in the relevant source files, a statement of the
+additional terms that apply to those files, or a notice indicating
+where to find the applicable terms.
+
+ Additional terms, permissive or non-permissive, may be stated in the
+form of a separately written license, or stated as exceptions;
+the above requirements apply either way.
+
+ 8. Termination.
+
+ You may not propagate or modify a covered work except as expressly
+provided under this License. Any attempt otherwise to propagate or
+modify it is void, and will automatically terminate your rights under
+this License (including any patent licenses granted under the third
+paragraph of section 11).
+
+ However, if you cease all violation of this License, then your
+license from a particular copyright holder is reinstated (a)
+provisionally, unless and until the copyright holder explicitly and
+finally terminates your license, and (b) permanently, if the copyright
+holder fails to notify you of the violation by some reasonable means
+prior to 60 days after the cessation.
+
+ Moreover, your license from a particular copyright holder is
+reinstated permanently if the copyright holder notifies you of the
+violation by some reasonable means, this is the first time you have
+received notice of violation of this License (for any work) from that
+copyright holder, and you cure the violation prior to 30 days after
+your receipt of the notice.
+
+ Termination of your rights under this section does not terminate the
+licenses of parties who have received copies or rights from you under
+this License. If your rights have been terminated and not permanently
+reinstated, you do not qualify to receive new licenses for the same
+material under section 10.
+
+ 9. Acceptance Not Required for Having Copies.
+
+ You are not required to accept this License in order to receive or
+run a copy of the Program. Ancillary propagation of a covered work
+occurring solely as a consequence of using peer-to-peer transmission
+to receive a copy likewise does not require acceptance. However,
+nothing other than this License grants you permission to propagate or
+modify any covered work. These actions infringe copyright if you do
+not accept this License. Therefore, by modifying or propagating a
+covered work, you indicate your acceptance of this License to do so.
+
+ 10. Automatic Licensing of Downstream Recipients.
+
+ Each time you convey a covered work, the recipient automatically
+receives a license from the original licensors, to run, modify and
+propagate that work, subject to this License. You are not responsible
+for enforcing compliance by third parties with this License.
+
+ An "entity transaction" is a transaction transferring control of an
+organization, or substantially all assets of one, or subdividing an
+organization, or merging organizations. If propagation of a covered
+work results from an entity transaction, each party to that
+transaction who receives a copy of the work also receives whatever
+licenses to the work the party's predecessor in interest had or could
+give under the previous paragraph, plus a right to possession of the
+Corresponding Source of the work from the predecessor in interest, if
+the predecessor has it or can get it with reasonable efforts.
+
+ You may not impose any further restrictions on the exercise of the
+rights granted or affirmed under this License. For example, you may
+not impose a license fee, royalty, or other charge for exercise of
+rights granted under this License, and you may not initiate litigation
+(including a cross-claim or counterclaim in a lawsuit) alleging that
+any patent claim is infringed by making, using, selling, offering for
+sale, or importing the Program or any portion of it.
+
+ 11. Patents.
+
+ A "contributor" is a copyright holder who authorizes use under this
+License of the Program or a work on which the Program is based. The
+work thus licensed is called the contributor's "contributor version".
+
+ A contributor's "essential patent claims" are all patent claims
+owned or controlled by the contributor, whether already acquired or
+hereafter acquired, that would be infringed by some manner, permitted
+by this License, of making, using, or selling its contributor version,
+but do not include claims that would be infringed only as a
+consequence of further modification of the contributor version. For
+purposes of this definition, "control" includes the right to grant
+patent sublicenses in a manner consistent with the requirements of
+this License.
+
+ Each contributor grants you a non-exclusive, worldwide, royalty-free
+patent license under the contributor's essential patent claims, to
+make, use, sell, offer for sale, import and otherwise run, modify and
+propagate the contents of its contributor version.
+
+ In the following three paragraphs, a "patent license" is any express
+agreement or commitment, however denominated, not to enforce a patent
+(such as an express permission to practice a patent or covenant not to
+sue for patent infringement). To "grant" such a patent license to a
+party means to make such an agreement or commitment not to enforce a
+patent against the party.
+
+ If you convey a covered work, knowingly relying on a patent license,
+and the Corresponding Source of the work is not available for anyone
+to copy, free of charge and under the terms of this License, through a
+publicly available network server or other readily accessible means,
+then you must either (1) cause the Corresponding Source to be so
+available, or (2) arrange to deprive yourself of the benefit of the
+patent license for this particular work, or (3) arrange, in a manner
+consistent with the requirements of this License, to extend the patent
+license to downstream recipients. "Knowingly relying" means you have
+actual knowledge that, but for the patent license, your conveying the
+covered work in a country, or your recipient's use of the covered work
+in a country, would infringe one or more identifiable patents in that
+country that you have reason to believe are valid.
+
+ If, pursuant to or in connection with a single transaction or
+arrangement, you convey, or propagate by procuring conveyance of, a
+covered work, and grant a patent license to some of the parties
+receiving the covered work authorizing them to use, propagate, modify
+or convey a specific copy of the covered work, then the patent license
+you grant is automatically extended to all recipients of the covered
+work and works based on it.
+
+ A patent license is "discriminatory" if it does not include within
+the scope of its coverage, prohibits the exercise of, or is
+conditioned on the non-exercise of one or more of the rights that are
+specifically granted under this License. You may not convey a covered
+work if you are a party to an arrangement with a third party that is
+in the business of distributing software, under which you make payment
+to the third party based on the extent of your activity of conveying
+the work, and under which the third party grants, to any of the
+parties who would receive the covered work from you, a discriminatory
+patent license (a) in connection with copies of the covered work
+conveyed by you (or copies made from those copies), or (b) primarily
+for and in connection with specific products or compilations that
+contain the covered work, unless you entered into that arrangement,
+or that patent license was granted, prior to 28 March 2007.
+
+ Nothing in this License shall be construed as excluding or limiting
+any implied license or other defenses to infringement that may
+otherwise be available to you under applicable patent law.
+
+ 12. No Surrender of Others' Freedom.
+
+ If conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License. If you cannot convey a
+covered work so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you may
+not convey it at all. For example, if you agree to terms that obligate you
+to collect a royalty for further conveying from those to whom you convey
+the Program, the only way you could satisfy both those terms and this
+License would be to refrain entirely from conveying the Program.
+
+ 13. Use with the GNU Affero General Public License.
+
+ Notwithstanding any other provision of this License, you have
+permission to link or combine any covered work with a work licensed
+under version 3 of the GNU Affero General Public License into a single
+combined work, and to convey the resulting work. The terms of this
+License will continue to apply to the part which is the covered work,
+but the special requirements of the GNU Affero General Public License,
+section 13, concerning interaction through a network will apply to the
+combination as such.
+
+ 14. Revised Versions of this License.
+
+ The Free Software Foundation may publish revised and/or new versions of
+the GNU General Public License from time to time. Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+ Each version is given a distinguishing version number. If the
+Program specifies that a certain numbered version of the GNU General
+Public License "or any later version" applies to it, you have the
+option of following the terms and conditions either of that numbered
+version or of any later version published by the Free Software
+Foundation. If the Program does not specify a version number of the
+GNU General Public License, you may choose any version ever published
+by the Free Software Foundation.
+
+ If the Program specifies that a proxy can decide which future
+versions of the GNU General Public License can be used, that proxy's
+public statement of acceptance of a version permanently authorizes you
+to choose that version for the Program.
+
+ Later license versions may give you additional or different
+permissions. However, no additional obligations are imposed on any
+author or copyright holder as a result of your choosing to follow a
+later version.
+
+ 15. Disclaimer of Warranty.
+
+ THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
+APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
+OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
+IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
+ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+ 16. Limitation of Liability.
+
+ IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
+THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
+USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
+DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
+PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
+EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGES.
+
+ 17. Interpretation of Sections 15 and 16.
+
+ If the disclaimer of warranty and limitation of liability provided
+above cannot be given local legal effect according to their terms,
+reviewing courts shall apply local law that most closely approximates
+an absolute waiver of all civil liability in connection with the
+Program, unless a warranty or assumption of liability accompanies a
+copy of the Program in return for a fee.
+
+ END OF TERMS AND CONDITIONS
+
+ How to Apply These Terms to Your New Programs
+
+ If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+ To do so, attach the following notices to the program. It is safest
+to attach them to the start of each source file to most effectively
+state the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+
+ Copyright (C)
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see .
+
+Also add information on how to contact you by electronic and paper mail.
+
+ If the program does terminal interaction, make it output a short
+notice like this when it starts in an interactive mode:
+
+ Copyright (C)
+ This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+ This is free software, and you are welcome to redistribute it
+ under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License. Of course, your program's commands
+might be different; for a GUI interface, you would use an "about box".
+
+ You should also get your employer (if you work as a programmer) or school,
+if any, to sign a "copyright disclaimer" for the program, if necessary.
+For more information on this, and how to apply and follow the GNU GPL, see
+.
+
+ The GNU General Public License does not permit incorporating your program
+into proprietary programs. If your program is a subroutine library, you
+may consider it more useful to permit linking proprietary applications with
+the library. If this is what you want to do, use the GNU Lesser General
+Public License instead of this License. But first, please read
+.
diff --git a/LICENSE.md b/LICENSE.md
new file mode 100644
index 0000000000000000000000000000000000000000..f288702d2fa16d3cdf0035b15a9fcbc552cd88e7
--- /dev/null
+++ b/LICENSE.md
@@ -0,0 +1,674 @@
+ GNU GENERAL PUBLIC LICENSE
+ Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc.
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+ Preamble
+
+ The GNU General Public License is a free, copyleft license for
+software and other kinds of works.
+
+ The licenses for most software and other practical works are designed
+to take away your freedom to share and change the works. By contrast,
+the GNU General Public License is intended to guarantee your freedom to
+share and change all versions of a program--to make sure it remains free
+software for all its users. We, the Free Software Foundation, use the
+GNU General Public License for most of our software; it applies also to
+any other work released this way by its authors. You can apply it to
+your programs, too.
+
+ When we speak of free software, we are referring to freedom, not
+price. Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+them if you wish), that you receive source code or can get it if you
+want it, that you can change the software or use pieces of it in new
+free programs, and that you know you can do these things.
+
+ To protect your rights, we need to prevent others from denying you
+these rights or asking you to surrender the rights. Therefore, you have
+certain responsibilities if you distribute copies of the software, or if
+you modify it: responsibilities to respect the freedom of others.
+
+ For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must pass on to the recipients the same
+freedoms that you received. You must make sure that they, too, receive
+or can get the source code. And you must show them these terms so they
+know their rights.
+
+ Developers that use the GNU GPL protect your rights with two steps:
+(1) assert copyright on the software, and (2) offer you this License
+giving you legal permission to copy, distribute and/or modify it.
+
+ For the developers' and authors' protection, the GPL clearly explains
+that there is no warranty for this free software. For both users' and
+authors' sake, the GPL requires that modified versions be marked as
+changed, so that their problems will not be attributed erroneously to
+authors of previous versions.
+
+ Some devices are designed to deny users access to install or run
+modified versions of the software inside them, although the manufacturer
+can do so. This is fundamentally incompatible with the aim of
+protecting users' freedom to change the software. The systematic
+pattern of such abuse occurs in the area of products for individuals to
+use, which is precisely where it is most unacceptable. Therefore, we
+have designed this version of the GPL to prohibit the practice for those
+products. If such problems arise substantially in other domains, we
+stand ready to extend this provision to those domains in future versions
+of the GPL, as needed to protect the freedom of users.
+
+ Finally, every program is threatened constantly by software patents.
+States should not allow patents to restrict development and use of
+software on general-purpose computers, but in those that do, we wish to
+avoid the special danger that patents applied to a free program could
+make it effectively proprietary. To prevent this, the GPL assures that
+patents cannot be used to render the program non-free.
+
+ The precise terms and conditions for copying, distribution and
+modification follow.
+
+ TERMS AND CONDITIONS
+
+ 0. Definitions.
+
+ "This License" refers to version 3 of the GNU General Public License.
+
+ "Copyright" also means copyright-like laws that apply to other kinds of
+works, such as semiconductor masks.
+
+ "The Program" refers to any copyrightable work licensed under this
+License. Each licensee is addressed as "you". "Licensees" and
+"recipients" may be individuals or organizations.
+
+ To "modify" a work means to copy from or adapt all or part of the work
+in a fashion requiring copyright permission, other than the making of an
+exact copy. The resulting work is called a "modified version" of the
+earlier work or a work "based on" the earlier work.
+
+ A "covered work" means either the unmodified Program or a work based
+on the Program.
+
+ To "propagate" a work means to do anything with it that, without
+permission, would make you directly or secondarily liable for
+infringement under applicable copyright law, except executing it on a
+computer or modifying a private copy. Propagation includes copying,
+distribution (with or without modification), making available to the
+public, and in some countries other activities as well.
+
+ To "convey" a work means any kind of propagation that enables other
+parties to make or receive copies. Mere interaction with a user through
+a computer network, with no transfer of a copy, is not conveying.
+
+ An interactive user interface displays "Appropriate Legal Notices"
+to the extent that it includes a convenient and prominently visible
+feature that (1) displays an appropriate copyright notice, and (2)
+tells the user that there is no warranty for the work (except to the
+extent that warranties are provided), that licensees may convey the
+work under this License, and how to view a copy of this License. If
+the interface presents a list of user commands or options, such as a
+menu, a prominent item in the list meets this criterion.
+
+ 1. Source Code.
+
+ The "source code" for a work means the preferred form of the work
+for making modifications to it. "Object code" means any non-source
+form of a work.
+
+ A "Standard Interface" means an interface that either is an official
+standard defined by a recognized standards body, or, in the case of
+interfaces specified for a particular programming language, one that
+is widely used among developers working in that language.
+
+ The "System Libraries" of an executable work include anything, other
+than the work as a whole, that (a) is included in the normal form of
+packaging a Major Component, but which is not part of that Major
+Component, and (b) serves only to enable use of the work with that
+Major Component, or to implement a Standard Interface for which an
+implementation is available to the public in source code form. A
+"Major Component", in this context, means a major essential component
+(kernel, window system, and so on) of the specific operating system
+(if any) on which the executable work runs, or a compiler used to
+produce the work, or an object code interpreter used to run it.
+
+ The "Corresponding Source" for a work in object code form means all
+the source code needed to generate, install, and (for an executable
+work) run the object code and to modify the work, including scripts to
+control those activities. However, it does not include the work's
+System Libraries, or general-purpose tools or generally available free
+programs which are used unmodified in performing those activities but
+which are not part of the work. For example, Corresponding Source
+includes interface definition files associated with source files for
+the work, and the source code for shared libraries and dynamically
+linked subprograms that the work is specifically designed to require,
+such as by intimate data communication or control flow between those
+subprograms and other parts of the work.
+
+ The Corresponding Source need not include anything that users
+can regenerate automatically from other parts of the Corresponding
+Source.
+
+ The Corresponding Source for a work in source code form is that
+same work.
+
+ 2. Basic Permissions.
+
+ All rights granted under this License are granted for the term of
+copyright on the Program, and are irrevocable provided the stated
+conditions are met. This License explicitly affirms your unlimited
+permission to run the unmodified Program. The output from running a
+covered work is covered by this License only if the output, given its
+content, constitutes a covered work. This License acknowledges your
+rights of fair use or other equivalent, as provided by copyright law.
+
+ You may make, run and propagate covered works that you do not
+convey, without conditions so long as your license otherwise remains
+in force. You may convey covered works to others for the sole purpose
+of having them make modifications exclusively for you, or provide you
+with facilities for running those works, provided that you comply with
+the terms of this License in conveying all material for which you do
+not control copyright. Those thus making or running the covered works
+for you must do so exclusively on your behalf, under your direction
+and control, on terms that prohibit them from making any copies of
+your copyrighted material outside their relationship with you.
+
+ Conveying under any other circumstances is permitted solely under
+the conditions stated below. Sublicensing is not allowed; section 10
+makes it unnecessary.
+
+ 3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+
+ No covered work shall be deemed part of an effective technological
+measure under any applicable law fulfilling obligations under article
+11 of the WIPO copyright treaty adopted on 20 December 1996, or
+similar laws prohibiting or restricting circumvention of such
+measures.
+
+ When you convey a covered work, you waive any legal power to forbid
+circumvention of technological measures to the extent such circumvention
+is effected by exercising rights under this License with respect to
+the covered work, and you disclaim any intention to limit operation or
+modification of the work as a means of enforcing, against the work's
+users, your or third parties' legal rights to forbid circumvention of
+technological measures.
+
+ 4. Conveying Verbatim Copies.
+
+ You may convey verbatim copies of the Program's source code as you
+receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice;
+keep intact all notices stating that this License and any
+non-permissive terms added in accord with section 7 apply to the code;
+keep intact all notices of the absence of any warranty; and give all
+recipients a copy of this License along with the Program.
+
+ You may charge any price or no price for each copy that you convey,
+and you may offer support or warranty protection for a fee.
+
+ 5. Conveying Modified Source Versions.
+
+ You may convey a work based on the Program, or the modifications to
+produce it from the Program, in the form of source code under the
+terms of section 4, provided that you also meet all of these conditions:
+
+ a) The work must carry prominent notices stating that you modified
+ it, and giving a relevant date.
+
+ b) The work must carry prominent notices stating that it is
+ released under this License and any conditions added under section
+ 7. This requirement modifies the requirement in section 4 to
+ "keep intact all notices".
+
+ c) You must license the entire work, as a whole, under this
+ License to anyone who comes into possession of a copy. This
+ License will therefore apply, along with any applicable section 7
+ additional terms, to the whole of the work, and all its parts,
+ regardless of how they are packaged. This License gives no
+ permission to license the work in any other way, but it does not
+ invalidate such permission if you have separately received it.
+
+ d) If the work has interactive user interfaces, each must display
+ Appropriate Legal Notices; however, if the Program has interactive
+ interfaces that do not display Appropriate Legal Notices, your
+ work need not make them do so.
+
+ A compilation of a covered work with other separate and independent
+works, which are not by their nature extensions of the covered work,
+and which are not combined with it such as to form a larger program,
+in or on a volume of a storage or distribution medium, is called an
+"aggregate" if the compilation and its resulting copyright are not
+used to limit the access or legal rights of the compilation's users
+beyond what the individual works permit. Inclusion of a covered work
+in an aggregate does not cause this License to apply to the other
+parts of the aggregate.
+
+ 6. Conveying Non-Source Forms.
+
+ You may convey a covered work in object code form under the terms
+of sections 4 and 5, provided that you also convey the
+machine-readable Corresponding Source under the terms of this License,
+in one of these ways:
+
+ a) Convey the object code in, or embodied in, a physical product
+ (including a physical distribution medium), accompanied by the
+ Corresponding Source fixed on a durable physical medium
+ customarily used for software interchange.
+
+ b) Convey the object code in, or embodied in, a physical product
+ (including a physical distribution medium), accompanied by a
+ written offer, valid for at least three years and valid for as
+ long as you offer spare parts or customer support for that product
+ model, to give anyone who possesses the object code either (1) a
+ copy of the Corresponding Source for all the software in the
+ product that is covered by this License, on a durable physical
+ medium customarily used for software interchange, for a price no
+ more than your reasonable cost of physically performing this
+ conveying of source, or (2) access to copy the
+ Corresponding Source from a network server at no charge.
+
+ c) Convey individual copies of the object code with a copy of the
+ written offer to provide the Corresponding Source. This
+ alternative is allowed only occasionally and noncommercially, and
+ only if you received the object code with such an offer, in accord
+ with subsection 6b.
+
+ d) Convey the object code by offering access from a designated
+ place (gratis or for a charge), and offer equivalent access to the
+ Corresponding Source in the same way through the same place at no
+ further charge. You need not require recipients to copy the
+ Corresponding Source along with the object code. If the place to
+ copy the object code is a network server, the Corresponding Source
+ may be on a different server (operated by you or a third party)
+ that supports equivalent copying facilities, provided you maintain
+ clear directions next to the object code saying where to find the
+ Corresponding Source. Regardless of what server hosts the
+ Corresponding Source, you remain obligated to ensure that it is
+ available for as long as needed to satisfy these requirements.
+
+ e) Convey the object code using peer-to-peer transmission, provided
+ you inform other peers where the object code and Corresponding
+ Source of the work are being offered to the general public at no
+ charge under subsection 6d.
+
+ A separable portion of the object code, whose source code is excluded
+from the Corresponding Source as a System Library, need not be
+included in conveying the object code work.
+
+ A "User Product" is either (1) a "consumer product", which means any
+tangible personal property which is normally used for personal, family,
+or household purposes, or (2) anything designed or sold for incorporation
+into a dwelling. In determining whether a product is a consumer product,
+doubtful cases shall be resolved in favor of coverage. For a particular
+product received by a particular user, "normally used" refers to a
+typical or common use of that class of product, regardless of the status
+of the particular user or of the way in which the particular user
+actually uses, or expects or is expected to use, the product. A product
+is a consumer product regardless of whether the product has substantial
+commercial, industrial or non-consumer uses, unless such uses represent
+the only significant mode of use of the product.
+
+ "Installation Information" for a User Product means any methods,
+procedures, authorization keys, or other information required to install
+and execute modified versions of a covered work in that User Product from
+a modified version of its Corresponding Source. The information must
+suffice to ensure that the continued functioning of the modified object
+code is in no case prevented or interfered with solely because
+modification has been made.
+
+ If you convey an object code work under this section in, or with, or
+specifically for use in, a User Product, and the conveying occurs as
+part of a transaction in which the right of possession and use of the
+User Product is transferred to the recipient in perpetuity or for a
+fixed term (regardless of how the transaction is characterized), the
+Corresponding Source conveyed under this section must be accompanied
+by the Installation Information. But this requirement does not apply
+if neither you nor any third party retains the ability to install
+modified object code on the User Product (for example, the work has
+been installed in ROM).
+
+ The requirement to provide Installation Information does not include a
+requirement to continue to provide support service, warranty, or updates
+for a work that has been modified or installed by the recipient, or for
+the User Product in which it has been modified or installed. Access to a
+network may be denied when the modification itself materially and
+adversely affects the operation of the network or violates the rules and
+protocols for communication across the network.
+
+ Corresponding Source conveyed, and Installation Information provided,
+in accord with this section must be in a format that is publicly
+documented (and with an implementation available to the public in
+source code form), and must require no special password or key for
+unpacking, reading or copying.
+
+ 7. Additional Terms.
+
+ "Additional permissions" are terms that supplement the terms of this
+License by making exceptions from one or more of its conditions.
+Additional permissions that are applicable to the entire Program shall
+be treated as though they were included in this License, to the extent
+that they are valid under applicable law. If additional permissions
+apply only to part of the Program, that part may be used separately
+under those permissions, but the entire Program remains governed by
+this License without regard to the additional permissions.
+
+ When you convey a copy of a covered work, you may at your option
+remove any additional permissions from that copy, or from any part of
+it. (Additional permissions may be written to require their own
+removal in certain cases when you modify the work.) You may place
+additional permissions on material, added by you to a covered work,
+for which you have or can give appropriate copyright permission.
+
+ Notwithstanding any other provision of this License, for material you
+add to a covered work, you may (if authorized by the copyright holders of
+that material) supplement the terms of this License with terms:
+
+ a) Disclaiming warranty or limiting liability differently from the
+ terms of sections 15 and 16 of this License; or
+
+ b) Requiring preservation of specified reasonable legal notices or
+ author attributions in that material or in the Appropriate Legal
+ Notices displayed by works containing it; or
+
+ c) Prohibiting misrepresentation of the origin of that material, or
+ requiring that modified versions of such material be marked in
+ reasonable ways as different from the original version; or
+
+ d) Limiting the use for publicity purposes of names of licensors or
+ authors of the material; or
+
+ e) Declining to grant rights under trademark law for use of some
+ trade names, trademarks, or service marks; or
+
+ f) Requiring indemnification of licensors and authors of that
+ material by anyone who conveys the material (or modified versions of
+ it) with contractual assumptions of liability to the recipient, for
+ any liability that these contractual assumptions directly impose on
+ those licensors and authors.
+
+ All other non-permissive additional terms are considered "further
+restrictions" within the meaning of section 10. If the Program as you
+received it, or any part of it, contains a notice stating that it is
+governed by this License along with a term that is a further
+restriction, you may remove that term. If a license document contains
+a further restriction but permits relicensing or conveying under this
+License, you may add to a covered work material governed by the terms
+of that license document, provided that the further restriction does
+not survive such relicensing or conveying.
+
+ If you add terms to a covered work in accord with this section, you
+must place, in the relevant source files, a statement of the
+additional terms that apply to those files, or a notice indicating
+where to find the applicable terms.
+
+ Additional terms, permissive or non-permissive, may be stated in the
+form of a separately written license, or stated as exceptions;
+the above requirements apply either way.
+
+ 8. Termination.
+
+ You may not propagate or modify a covered work except as expressly
+provided under this License. Any attempt otherwise to propagate or
+modify it is void, and will automatically terminate your rights under
+this License (including any patent licenses granted under the third
+paragraph of section 11).
+
+ However, if you cease all violation of this License, then your
+license from a particular copyright holder is reinstated (a)
+provisionally, unless and until the copyright holder explicitly and
+finally terminates your license, and (b) permanently, if the copyright
+holder fails to notify you of the violation by some reasonable means
+prior to 60 days after the cessation.
+
+ Moreover, your license from a particular copyright holder is
+reinstated permanently if the copyright holder notifies you of the
+violation by some reasonable means, this is the first time you have
+received notice of violation of this License (for any work) from that
+copyright holder, and you cure the violation prior to 30 days after
+your receipt of the notice.
+
+ Termination of your rights under this section does not terminate the
+licenses of parties who have received copies or rights from you under
+this License. If your rights have been terminated and not permanently
+reinstated, you do not qualify to receive new licenses for the same
+material under section 10.
+
+ 9. Acceptance Not Required for Having Copies.
+
+ You are not required to accept this License in order to receive or
+run a copy of the Program. Ancillary propagation of a covered work
+occurring solely as a consequence of using peer-to-peer transmission
+to receive a copy likewise does not require acceptance. However,
+nothing other than this License grants you permission to propagate or
+modify any covered work. These actions infringe copyright if you do
+not accept this License. Therefore, by modifying or propagating a
+covered work, you indicate your acceptance of this License to do so.
+
+ 10. Automatic Licensing of Downstream Recipients.
+
+ Each time you convey a covered work, the recipient automatically
+receives a license from the original licensors, to run, modify and
+propagate that work, subject to this License. You are not responsible
+for enforcing compliance by third parties with this License.
+
+ An "entity transaction" is a transaction transferring control of an
+organization, or substantially all assets of one, or subdividing an
+organization, or merging organizations. If propagation of a covered
+work results from an entity transaction, each party to that
+transaction who receives a copy of the work also receives whatever
+licenses to the work the party's predecessor in interest had or could
+give under the previous paragraph, plus a right to possession of the
+Corresponding Source of the work from the predecessor in interest, if
+the predecessor has it or can get it with reasonable efforts.
+
+ You may not impose any further restrictions on the exercise of the
+rights granted or affirmed under this License. For example, you may
+not impose a license fee, royalty, or other charge for exercise of
+rights granted under this License, and you may not initiate litigation
+(including a cross-claim or counterclaim in a lawsuit) alleging that
+any patent claim is infringed by making, using, selling, offering for
+sale, or importing the Program or any portion of it.
+
+ 11. Patents.
+
+ A "contributor" is a copyright holder who authorizes use under this
+License of the Program or a work on which the Program is based. The
+work thus licensed is called the contributor's "contributor version".
+
+ A contributor's "essential patent claims" are all patent claims
+owned or controlled by the contributor, whether already acquired or
+hereafter acquired, that would be infringed by some manner, permitted
+by this License, of making, using, or selling its contributor version,
+but do not include claims that would be infringed only as a
+consequence of further modification of the contributor version. For
+purposes of this definition, "control" includes the right to grant
+patent sublicenses in a manner consistent with the requirements of
+this License.
+
+ Each contributor grants you a non-exclusive, worldwide, royalty-free
+patent license under the contributor's essential patent claims, to
+make, use, sell, offer for sale, import and otherwise run, modify and
+propagate the contents of its contributor version.
+
+ In the following three paragraphs, a "patent license" is any express
+agreement or commitment, however denominated, not to enforce a patent
+(such as an express permission to practice a patent or covenant not to
+sue for patent infringement). To "grant" such a patent license to a
+party means to make such an agreement or commitment not to enforce a
+patent against the party.
+
+ If you convey a covered work, knowingly relying on a patent license,
+and the Corresponding Source of the work is not available for anyone
+to copy, free of charge and under the terms of this License, through a
+publicly available network server or other readily accessible means,
+then you must either (1) cause the Corresponding Source to be so
+available, or (2) arrange to deprive yourself of the benefit of the
+patent license for this particular work, or (3) arrange, in a manner
+consistent with the requirements of this License, to extend the patent
+license to downstream recipients. "Knowingly relying" means you have
+actual knowledge that, but for the patent license, your conveying the
+covered work in a country, or your recipient's use of the covered work
+in a country, would infringe one or more identifiable patents in that
+country that you have reason to believe are valid.
+
+ If, pursuant to or in connection with a single transaction or
+arrangement, you convey, or propagate by procuring conveyance of, a
+covered work, and grant a patent license to some of the parties
+receiving the covered work authorizing them to use, propagate, modify
+or convey a specific copy of the covered work, then the patent license
+you grant is automatically extended to all recipients of the covered
+work and works based on it.
+
+ A patent license is "discriminatory" if it does not include within
+the scope of its coverage, prohibits the exercise of, or is
+conditioned on the non-exercise of one or more of the rights that are
+specifically granted under this License. You may not convey a covered
+work if you are a party to an arrangement with a third party that is
+in the business of distributing software, under which you make payment
+to the third party based on the extent of your activity of conveying
+the work, and under which the third party grants, to any of the
+parties who would receive the covered work from you, a discriminatory
+patent license (a) in connection with copies of the covered work
+conveyed by you (or copies made from those copies), or (b) primarily
+for and in connection with specific products or compilations that
+contain the covered work, unless you entered into that arrangement,
+or that patent license was granted, prior to 28 March 2007.
+
+ Nothing in this License shall be construed as excluding or limiting
+any implied license or other defenses to infringement that may
+otherwise be available to you under applicable patent law.
+
+ 12. No Surrender of Others' Freedom.
+
+ If conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License. If you cannot convey a
+covered work so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you may
+not convey it at all. For example, if you agree to terms that obligate you
+to collect a royalty for further conveying from those to whom you convey
+the Program, the only way you could satisfy both those terms and this
+License would be to refrain entirely from conveying the Program.
+
+ 13. Use with the GNU Affero General Public License.
+
+ Notwithstanding any other provision of this License, you have
+permission to link or combine any covered work with a work licensed
+under version 3 of the GNU Affero General Public License into a single
+combined work, and to convey the resulting work. The terms of this
+License will continue to apply to the part which is the covered work,
+but the special requirements of the GNU Affero General Public License,
+section 13, concerning interaction through a network will apply to the
+combination as such.
+
+ 14. Revised Versions of this License.
+
+ The Free Software Foundation may publish revised and/or new versions of
+the GNU General Public License from time to time. Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+ Each version is given a distinguishing version number. If the
+Program specifies that a certain numbered version of the GNU General
+Public License "or any later version" applies to it, you have the
+option of following the terms and conditions either of that numbered
+version or of any later version published by the Free Software
+Foundation. If the Program does not specify a version number of the
+GNU General Public License, you may choose any version ever published
+by the Free Software Foundation.
+
+ If the Program specifies that a proxy can decide which future
+versions of the GNU General Public License can be used, that proxy's
+public statement of acceptance of a version permanently authorizes you
+to choose that version for the Program.
+
+ Later license versions may give you additional or different
+permissions. However, no additional obligations are imposed on any
+author or copyright holder as a result of your choosing to follow a
+later version.
+
+ 15. Disclaimer of Warranty.
+
+ THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
+APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
+OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
+IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
+ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+ 16. Limitation of Liability.
+
+ IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
+THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
+USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
+DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
+PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
+EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGES.
+
+ 17. Interpretation of Sections 15 and 16.
+
+ If the disclaimer of warranty and limitation of liability provided
+above cannot be given local legal effect according to their terms,
+reviewing courts shall apply local law that most closely approximates
+an absolute waiver of all civil liability in connection with the
+Program, unless a warranty or assumption of liability accompanies a
+copy of the Program in return for a fee.
+
+ END OF TERMS AND CONDITIONS
+
+ How to Apply These Terms to Your New Programs
+
+ If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+ To do so, attach the following notices to the program. It is safest
+to attach them to the start of each source file to most effectively
+state the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+
+ Copyright (C)
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see .
+
+Also add information on how to contact you by electronic and paper mail.
+
+ If the program does terminal interaction, make it output a short
+notice like this when it starts in an interactive mode:
+
+ Copyright (C)
+ This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+ This is free software, and you are welcome to redistribute it
+ under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License. Of course, your program's commands
+might be different; for a GUI interface, you would use an "about box".
+
+ You should also get your employer (if you work as a programmer) or school,
+if any, to sign a "copyright disclaimer" for the program, if necessary.
+For more information on this, and how to apply and follow the GNU GPL, see
+.
+
+ The GNU General Public License does not permit incorporating your program
+into proprietary programs. If your program is a subroutine library, you
+may consider it more useful to permit linking proprietary applications with
+the library. If this is what you want to do, use the GNU Lesser General
+Public License instead of this License. But first, please read
+.
diff --git a/app.py b/app.py
new file mode 100644
index 0000000000000000000000000000000000000000..2eacb0eb85feb6e74ab4fb57c97909b39e9cd38b
--- /dev/null
+++ b/app.py
@@ -0,0 +1,66 @@
+
+import subprocess
+
+import streamlit as st
+import matplotlib.pyplot as plt
+import matplotlib.image as mpimg
+import subprocess
+import os
+from PIL import Image
+import torch
+import sys
+#import cv2
+
+
+def add_logo(logo_path, size=(200, 150)):
+ logo = Image.open('logoAI.png')
+ logo = logo.resize(size)
+ st.image(logo, use_column_width=False)
+
+def run_detection(image_path):
+ env = os.environ.copy()
+ env['PYTHONPATH'] = '/mount/src/yolo9tr/'
+
+ # Run the detection command
+ command = [
+ "python", "detect_dual.py",
+ "--source", image_path,
+ "--img", "640",
+ "--device", "cpu",
+ "--weights", "models/detect/yolov9tr.pt",
+ "--name", "yolov9_c_640_detect",
+ "--exist-ok"
+ ]
+ subprocess.run(command, check=True, env=os.environ)
+
+ # Find the output image
+ output_dir = "runs/detect/yolov9_c_640_detect"
+ output_image = os.path.join(output_dir, os.path.basename(image_path))
+ return output_image
+
+def main():
+ st.title("YOLO9tr Object Detection")
+
+ # Add the research center logo at the top of the app
+ add_logo("research_center_logo.png")
+
+ uploaded_file = st.file_uploader("Choose an image...", type=["jpg", "jpeg", "png"])
+
+ if uploaded_file is not None:
+ image_path = "temp_image.jpg"
+ with open(image_path, "wb") as f:
+ f.write(uploaded_file.getbuffer())
+ else:
+ image_path = "United_States_000502.jpg" # Default image
+
+ st.image(image_path, caption="Image for Detection", use_column_width=True)
+
+ if st.button("Run Detection"):
+ with st.spinner("Running detection..."):
+ output_image = run_detection(image_path)
+
+ # Display the output image
+ st.image(output_image, caption="Detection Result", use_column_width=True)
+
+if __name__ == "__main__":
+ main()
\ No newline at end of file
diff --git a/app2.py b/app2.py
new file mode 100644
index 0000000000000000000000000000000000000000..173804cd81dd3a240d1196cd3dad383a8fbe34e3
--- /dev/null
+++ b/app2.py
@@ -0,0 +1,60 @@
+import gradio as gr
+import subprocess
+import os
+from PIL import Image
+import torch
+
+def add_logo(img):
+ logo = Image.open('logoAI.png')
+ logo = logo.resize((200, 150))
+ img_with_logo = Image.new('RGB', (img.width, img.height + logo.height))
+ img_with_logo.paste(logo, (0, 0))
+ img_with_logo.paste(img, (0, logo.height))
+ return img_with_logo
+
+def run_detection(image):
+ # Save the uploaded image temporarily
+ image_path = "temp_image.jpg"
+ image.save(image_path)
+
+ env = os.environ.copy()
+ env['PYTHONPATH'] = '/mount/src/yolo9tr/'
+
+ # Run the detection command
+ command = [
+ "python", "detect_dual.py",
+ "--source", image_path,
+ "--img", "640",
+ "--device", "cpu",
+ "--weights", "models/detect/yolov9tr.pt",
+ "--name", "yolov9_c_640_detect",
+ "--exist-ok"
+ ]
+ subprocess.run(command, check=True, env=os.environ)
+
+ # Find the output image
+ output_dir = "runs/detect/yolov9_c_640_detect"
+ output_image = os.path.join(output_dir, os.path.basename(image_path))
+
+ # Add logo to the output image
+ output_with_logo = add_logo(Image.open(output_image))
+
+ return output_with_logo
+
+def main():
+ input_image = gr.Image(type="pil", label="Upload an image")
+ output_image = gr.Image(type="pil", label="Detection Result")
+
+ iface = gr.Interface(
+ fn=run_detection,
+ inputs=input_image,
+ outputs=output_image,
+ title="YOLO9tr Object Detection",
+ description="Upload an image to perform object detection using YOLO9tr.",
+ examples=[["United_States_000502.jpg"]]
+ )
+
+ iface.launch()
+
+if __name__ == "__main__":
+ main()
\ No newline at end of file
diff --git a/benchmarks.py b/benchmarks.py
new file mode 100644
index 0000000000000000000000000000000000000000..462636b25ef3b0ea6aa804abe751b0b1de765864
--- /dev/null
+++ b/benchmarks.py
@@ -0,0 +1,142 @@
+import argparse
+import platform
+import sys
+import time
+from pathlib import Path
+
+import pandas as pd
+
+FILE = Path(__file__).resolve()
+ROOT = FILE.parents[0] # YOLO root directory
+if str(ROOT) not in sys.path:
+ sys.path.append(str(ROOT)) # add ROOT to PATH
+# ROOT = ROOT.relative_to(Path.cwd()) # relative
+
+import export
+from models.experimental import attempt_load
+from models.yolo import SegmentationModel
+from segment.val import run as val_seg
+from utils import notebook_init
+from utils.general import LOGGER, check_yaml, file_size, print_args
+from utils.torch_utils import select_device
+from val import run as val_det
+
+
+def run(
+ weights=ROOT / 'yolo.pt', # weights path
+ imgsz=640, # inference size (pixels)
+ batch_size=1, # batch size
+ data=ROOT / 'data/coco.yaml', # dataset.yaml path
+ device='', # cuda device, i.e. 0 or 0,1,2,3 or cpu
+ half=False, # use FP16 half-precision inference
+ test=False, # test exports only
+ pt_only=False, # test PyTorch only
+ hard_fail=False, # throw error on benchmark failure
+):
+ y, t = [], time.time()
+ device = select_device(device)
+ model_type = type(attempt_load(weights, fuse=False)) # DetectionModel, SegmentationModel, etc.
+ for i, (name, f, suffix, cpu, gpu) in export.export_formats().iterrows(): # index, (name, file, suffix, CPU, GPU)
+ try:
+ assert i not in (9, 10), 'inference not supported' # Edge TPU and TF.js are unsupported
+ assert i != 5 or platform.system() == 'Darwin', 'inference only supported on macOS>=10.13' # CoreML
+ if 'cpu' in device.type:
+ assert cpu, 'inference not supported on CPU'
+ if 'cuda' in device.type:
+ assert gpu, 'inference not supported on GPU'
+
+ # Export
+ if f == '-':
+ w = weights # PyTorch format
+ else:
+ w = export.run(weights=weights, imgsz=[imgsz], include=[f], device=device, half=half)[-1] # all others
+ assert suffix in str(w), 'export failed'
+
+ # Validate
+ if model_type == SegmentationModel:
+ result = val_seg(data, w, batch_size, imgsz, plots=False, device=device, task='speed', half=half)
+ metric = result[0][7] # (box(p, r, map50, map), mask(p, r, map50, map), *loss(box, obj, cls))
+ else: # DetectionModel:
+ result = val_det(data, w, batch_size, imgsz, plots=False, device=device, task='speed', half=half)
+ metric = result[0][3] # (p, r, map50, map, *loss(box, obj, cls))
+ speed = result[2][1] # times (preprocess, inference, postprocess)
+ y.append([name, round(file_size(w), 1), round(metric, 4), round(speed, 2)]) # MB, mAP, t_inference
+ except Exception as e:
+ if hard_fail:
+ assert type(e) is AssertionError, f'Benchmark --hard-fail for {name}: {e}'
+ LOGGER.warning(f'WARNING ⚠️ Benchmark failure for {name}: {e}')
+ y.append([name, None, None, None]) # mAP, t_inference
+ if pt_only and i == 0:
+ break # break after PyTorch
+
+ # Print results
+ LOGGER.info('\n')
+ parse_opt()
+ notebook_init() # print system info
+ c = ['Format', 'Size (MB)', 'mAP50-95', 'Inference time (ms)'] if map else ['Format', 'Export', '', '']
+ py = pd.DataFrame(y, columns=c)
+ LOGGER.info(f'\nBenchmarks complete ({time.time() - t:.2f}s)')
+ LOGGER.info(str(py if map else py.iloc[:, :2]))
+ if hard_fail and isinstance(hard_fail, str):
+ metrics = py['mAP50-95'].array # values to compare to floor
+ floor = eval(hard_fail) # minimum metric floor to pass
+ assert all(x > floor for x in metrics if pd.notna(x)), f'HARD FAIL: mAP50-95 < floor {floor}'
+ return py
+
+
+def test(
+ weights=ROOT / 'yolo.pt', # weights path
+ imgsz=640, # inference size (pixels)
+ batch_size=1, # batch size
+ data=ROOT / 'data/coco128.yaml', # dataset.yaml path
+ device='', # cuda device, i.e. 0 or 0,1,2,3 or cpu
+ half=False, # use FP16 half-precision inference
+ test=False, # test exports only
+ pt_only=False, # test PyTorch only
+ hard_fail=False, # throw error on benchmark failure
+):
+ y, t = [], time.time()
+ device = select_device(device)
+ for i, (name, f, suffix, gpu) in export.export_formats().iterrows(): # index, (name, file, suffix, gpu-capable)
+ try:
+ w = weights if f == '-' else \
+ export.run(weights=weights, imgsz=[imgsz], include=[f], device=device, half=half)[-1] # weights
+ assert suffix in str(w), 'export failed'
+ y.append([name, True])
+ except Exception:
+ y.append([name, False]) # mAP, t_inference
+
+ # Print results
+ LOGGER.info('\n')
+ parse_opt()
+ notebook_init() # print system info
+ py = pd.DataFrame(y, columns=['Format', 'Export'])
+ LOGGER.info(f'\nExports complete ({time.time() - t:.2f}s)')
+ LOGGER.info(str(py))
+ return py
+
+
+def parse_opt():
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--weights', type=str, default=ROOT / 'yolo.pt', help='weights path')
+ parser.add_argument('--imgsz', '--img', '--img-size', type=int, default=640, help='inference size (pixels)')
+ parser.add_argument('--batch-size', type=int, default=1, help='batch size')
+ parser.add_argument('--data', type=str, default=ROOT / 'data/coco128.yaml', help='dataset.yaml path')
+ parser.add_argument('--device', default='', help='cuda device, i.e. 0 or 0,1,2,3 or cpu')
+ parser.add_argument('--half', action='store_true', help='use FP16 half-precision inference')
+ parser.add_argument('--test', action='store_true', help='test exports only')
+ parser.add_argument('--pt-only', action='store_true', help='test PyTorch only')
+ parser.add_argument('--hard-fail', nargs='?', const=True, default=False, help='Exception on error or < min metric')
+ opt = parser.parse_args()
+ opt.data = check_yaml(opt.data) # check YAML
+ print_args(vars(opt))
+ return opt
+
+
+def main(opt):
+ test(**vars(opt)) if opt.test else run(**vars(opt))
+
+
+if __name__ == "__main__":
+ opt = parse_opt()
+ main(opt)
diff --git a/classify/predict.py b/classify/predict.py
new file mode 100644
index 0000000000000000000000000000000000000000..9a6b0006293202dc2193edac6f809cfe8a132062
--- /dev/null
+++ b/classify/predict.py
@@ -0,0 +1,224 @@
+# YOLOv5 🚀 by Ultralytics, GPL-3.0 license
+"""
+Run YOLOv5 classification inference on images, videos, directories, globs, YouTube, webcam, streams, etc.
+
+Usage - sources:
+ $ python classify/predict.py --weights yolov5s-cls.pt --source 0 # webcam
+ img.jpg # image
+ vid.mp4 # video
+ screen # screenshot
+ path/ # directory
+ 'path/*.jpg' # glob
+ 'https://youtu.be/Zgi9g1ksQHc' # YouTube
+ 'rtsp://example.com/media.mp4' # RTSP, RTMP, HTTP stream
+
+Usage - formats:
+ $ python classify/predict.py --weights yolov5s-cls.pt # PyTorch
+ yolov5s-cls.torchscript # TorchScript
+ yolov5s-cls.onnx # ONNX Runtime or OpenCV DNN with --dnn
+ yolov5s-cls_openvino_model # OpenVINO
+ yolov5s-cls.engine # TensorRT
+ yolov5s-cls.mlmodel # CoreML (macOS-only)
+ yolov5s-cls_saved_model # TensorFlow SavedModel
+ yolov5s-cls.pb # TensorFlow GraphDef
+ yolov5s-cls.tflite # TensorFlow Lite
+ yolov5s-cls_edgetpu.tflite # TensorFlow Edge TPU
+ yolov5s-cls_paddle_model # PaddlePaddle
+"""
+
+import argparse
+import os
+import platform
+import sys
+from pathlib import Path
+
+import torch
+import torch.nn.functional as F
+
+FILE = Path(__file__).resolve()
+ROOT = FILE.parents[1] # YOLOv5 root directory
+if str(ROOT) not in sys.path:
+ sys.path.append(str(ROOT)) # add ROOT to PATH
+ROOT = Path(os.path.relpath(ROOT, Path.cwd())) # relative
+
+from models.common import DetectMultiBackend
+from utils.augmentations import classify_transforms
+from utils.dataloaders import IMG_FORMATS, VID_FORMATS, LoadImages, LoadScreenshots, LoadStreams
+from utils.general import (LOGGER, Profile, check_file, check_img_size, check_imshow, check_requirements, colorstr, cv2,
+ increment_path, print_args, strip_optimizer)
+from utils.plots import Annotator
+from utils.torch_utils import select_device, smart_inference_mode
+
+
+@smart_inference_mode()
+def run(
+ weights=ROOT / 'yolov5s-cls.pt', # model.pt path(s)
+ source=ROOT / 'data/images', # file/dir/URL/glob/screen/0(webcam)
+ data=ROOT / 'data/coco128.yaml', # dataset.yaml path
+ imgsz=(224, 224), # inference size (height, width)
+ device='', # cuda device, i.e. 0 or 0,1,2,3 or cpu
+ view_img=False, # show results
+ save_txt=False, # save results to *.txt
+ nosave=False, # do not save images/videos
+ augment=False, # augmented inference
+ visualize=False, # visualize features
+ update=False, # update all models
+ project=ROOT / 'runs/predict-cls', # save results to project/name
+ name='exp', # save results to project/name
+ exist_ok=False, # existing project/name ok, do not increment
+ half=False, # use FP16 half-precision inference
+ dnn=False, # use OpenCV DNN for ONNX inference
+ vid_stride=1, # video frame-rate stride
+):
+ source = str(source)
+ save_img = not nosave and not source.endswith('.txt') # save inference images
+ is_file = Path(source).suffix[1:] in (IMG_FORMATS + VID_FORMATS)
+ is_url = source.lower().startswith(('rtsp://', 'rtmp://', 'http://', 'https://'))
+ webcam = source.isnumeric() or source.endswith('.txt') or (is_url and not is_file)
+ screenshot = source.lower().startswith('screen')
+ if is_url and is_file:
+ source = check_file(source) # download
+
+ # Directories
+ save_dir = increment_path(Path(project) / name, exist_ok=exist_ok) # increment run
+ (save_dir / 'labels' if save_txt else save_dir).mkdir(parents=True, exist_ok=True) # make dir
+
+ # Load model
+ device = select_device(device)
+ model = DetectMultiBackend(weights, device=device, dnn=dnn, data=data, fp16=half)
+ stride, names, pt = model.stride, model.names, model.pt
+ imgsz = check_img_size(imgsz, s=stride) # check image size
+
+ # Dataloader
+ bs = 1 # batch_size
+ if webcam:
+ view_img = check_imshow(warn=True)
+ dataset = LoadStreams(source, img_size=imgsz, transforms=classify_transforms(imgsz[0]), vid_stride=vid_stride)
+ bs = len(dataset)
+ elif screenshot:
+ dataset = LoadScreenshots(source, img_size=imgsz, stride=stride, auto=pt)
+ else:
+ dataset = LoadImages(source, img_size=imgsz, transforms=classify_transforms(imgsz[0]), vid_stride=vid_stride)
+ vid_path, vid_writer = [None] * bs, [None] * bs
+
+ # Run inference
+ model.warmup(imgsz=(1 if pt else bs, 3, *imgsz)) # warmup
+ seen, windows, dt = 0, [], (Profile(), Profile(), Profile())
+ for path, im, im0s, vid_cap, s in dataset:
+ with dt[0]:
+ im = torch.Tensor(im).to(model.device)
+ im = im.half() if model.fp16 else im.float() # uint8 to fp16/32
+ if len(im.shape) == 3:
+ im = im[None] # expand for batch dim
+
+ # Inference
+ with dt[1]:
+ results = model(im)
+
+ # Post-process
+ with dt[2]:
+ pred = F.softmax(results, dim=1) # probabilities
+
+ # Process predictions
+ for i, prob in enumerate(pred): # per image
+ seen += 1
+ if webcam: # batch_size >= 1
+ p, im0, frame = path[i], im0s[i].copy(), dataset.count
+ s += f'{i}: '
+ else:
+ p, im0, frame = path, im0s.copy(), getattr(dataset, 'frame', 0)
+
+ p = Path(p) # to Path
+ save_path = str(save_dir / p.name) # im.jpg
+ txt_path = str(save_dir / 'labels' / p.stem) + ('' if dataset.mode == 'image' else f'_{frame}') # im.txt
+
+ s += '%gx%g ' % im.shape[2:] # print string
+ annotator = Annotator(im0, example=str(names), pil=True)
+
+ # Print results
+ top5i = prob.argsort(0, descending=True)[:5].tolist() # top 5 indices
+ s += f"{', '.join(f'{names[j]} {prob[j]:.2f}' for j in top5i)}, "
+
+ # Write results
+ text = '\n'.join(f'{prob[j]:.2f} {names[j]}' for j in top5i)
+ if save_img or view_img: # Add bbox to image
+ annotator.text((32, 32), text, txt_color=(255, 255, 255))
+ if save_txt: # Write to file
+ with open(f'{txt_path}.txt', 'a') as f:
+ f.write(text + '\n')
+
+ # Stream results
+ im0 = annotator.result()
+ if view_img:
+ if platform.system() == 'Linux' and p not in windows:
+ windows.append(p)
+ cv2.namedWindow(str(p), cv2.WINDOW_NORMAL | cv2.WINDOW_KEEPRATIO) # allow window resize (Linux)
+ cv2.resizeWindow(str(p), im0.shape[1], im0.shape[0])
+ cv2.imshow(str(p), im0)
+ cv2.waitKey(1) # 1 millisecond
+
+ # Save results (image with detections)
+ if save_img:
+ if dataset.mode == 'image':
+ cv2.imwrite(save_path, im0)
+ else: # 'video' or 'stream'
+ if vid_path[i] != save_path: # new video
+ vid_path[i] = save_path
+ if isinstance(vid_writer[i], cv2.VideoWriter):
+ vid_writer[i].release() # release previous video writer
+ if vid_cap: # video
+ fps = vid_cap.get(cv2.CAP_PROP_FPS)
+ w = int(vid_cap.get(cv2.CAP_PROP_FRAME_WIDTH))
+ h = int(vid_cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
+ else: # stream
+ fps, w, h = 30, im0.shape[1], im0.shape[0]
+ save_path = str(Path(save_path).with_suffix('.mp4')) # force *.mp4 suffix on results videos
+ vid_writer[i] = cv2.VideoWriter(save_path, cv2.VideoWriter_fourcc(*'mp4v'), fps, (w, h))
+ vid_writer[i].write(im0)
+
+ # Print time (inference-only)
+ LOGGER.info(f"{s}{dt[1].dt * 1E3:.1f}ms")
+
+ # Print results
+ t = tuple(x.t / seen * 1E3 for x in dt) # speeds per image
+ LOGGER.info(f'Speed: %.1fms pre-process, %.1fms inference, %.1fms NMS per image at shape {(1, 3, *imgsz)}' % t)
+ if save_txt or save_img:
+ s = f"\n{len(list(save_dir.glob('labels/*.txt')))} labels saved to {save_dir / 'labels'}" if save_txt else ''
+ LOGGER.info(f"Results saved to {colorstr('bold', save_dir)}{s}")
+ if update:
+ strip_optimizer(weights[0]) # update model (to fix SourceChangeWarning)
+
+
+def parse_opt():
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--weights', nargs='+', type=str, default=ROOT / 'yolov5s-cls.pt', help='model path(s)')
+ parser.add_argument('--source', type=str, default=ROOT / 'data/images', help='file/dir/URL/glob/screen/0(webcam)')
+ parser.add_argument('--data', type=str, default=ROOT / 'data/coco128.yaml', help='(optional) dataset.yaml path')
+ parser.add_argument('--imgsz', '--img', '--img-size', nargs='+', type=int, default=[224], help='inference size h,w')
+ parser.add_argument('--device', default='', help='cuda device, i.e. 0 or 0,1,2,3 or cpu')
+ parser.add_argument('--view-img', action='store_true', help='show results')
+ parser.add_argument('--save-txt', action='store_true', help='save results to *.txt')
+ parser.add_argument('--nosave', action='store_true', help='do not save images/videos')
+ parser.add_argument('--augment', action='store_true', help='augmented inference')
+ parser.add_argument('--visualize', action='store_true', help='visualize features')
+ parser.add_argument('--update', action='store_true', help='update all models')
+ parser.add_argument('--project', default=ROOT / 'runs/predict-cls', help='save results to project/name')
+ parser.add_argument('--name', default='exp', help='save results to project/name')
+ parser.add_argument('--exist-ok', action='store_true', help='existing project/name ok, do not increment')
+ parser.add_argument('--half', action='store_true', help='use FP16 half-precision inference')
+ parser.add_argument('--dnn', action='store_true', help='use OpenCV DNN for ONNX inference')
+ parser.add_argument('--vid-stride', type=int, default=1, help='video frame-rate stride')
+ opt = parser.parse_args()
+ opt.imgsz *= 2 if len(opt.imgsz) == 1 else 1 # expand
+ print_args(vars(opt))
+ return opt
+
+
+def main(opt):
+ check_requirements(exclude=('tensorboard', 'thop'))
+ run(**vars(opt))
+
+
+if __name__ == "__main__":
+ opt = parse_opt()
+ main(opt)
diff --git a/classify/train.py b/classify/train.py
new file mode 100644
index 0000000000000000000000000000000000000000..a50845a4f781e5953567cd7e0304b81ca320c6d3
--- /dev/null
+++ b/classify/train.py
@@ -0,0 +1,333 @@
+# YOLOv5 🚀 by Ultralytics, GPL-3.0 license
+"""
+Train a YOLOv5 classifier model on a classification dataset
+
+Usage - Single-GPU training:
+ $ python classify/train.py --model yolov5s-cls.pt --data imagenette160 --epochs 5 --img 224
+
+Usage - Multi-GPU DDP training:
+ $ python -m torch.distributed.run --nproc_per_node 4 --master_port 1 classify/train.py --model yolov5s-cls.pt --data imagenet --epochs 5 --img 224 --device 0,1,2,3
+
+Datasets: --data mnist, fashion-mnist, cifar10, cifar100, imagenette, imagewoof, imagenet, or 'path/to/data'
+YOLOv5-cls models: --model yolov5n-cls.pt, yolov5s-cls.pt, yolov5m-cls.pt, yolov5l-cls.pt, yolov5x-cls.pt
+Torchvision models: --model resnet50, efficientnet_b0, etc. See https://pytorch.org/vision/stable/models.html
+"""
+
+import argparse
+import os
+import subprocess
+import sys
+import time
+from copy import deepcopy
+from datetime import datetime
+from pathlib import Path
+
+import torch
+import torch.distributed as dist
+import torch.hub as hub
+import torch.optim.lr_scheduler as lr_scheduler
+import torchvision
+from torch.cuda import amp
+from tqdm import tqdm
+
+FILE = Path(__file__).resolve()
+ROOT = FILE.parents[1] # YOLOv5 root directory
+if str(ROOT) not in sys.path:
+ sys.path.append(str(ROOT)) # add ROOT to PATH
+ROOT = Path(os.path.relpath(ROOT, Path.cwd())) # relative
+
+from classify import val as validate
+from models.experimental import attempt_load
+from models.yolo import ClassificationModel, DetectionModel
+from utils.dataloaders import create_classification_dataloader
+from utils.general import (DATASETS_DIR, LOGGER, TQDM_BAR_FORMAT, WorkingDirectory, check_git_info, check_git_status,
+ check_requirements, colorstr, download, increment_path, init_seeds, print_args, yaml_save)
+from utils.loggers import GenericLogger
+from utils.plots import imshow_cls
+from utils.torch_utils import (ModelEMA, model_info, reshape_classifier_output, select_device, smart_DDP,
+ smart_optimizer, smartCrossEntropyLoss, torch_distributed_zero_first)
+
+LOCAL_RANK = int(os.getenv('LOCAL_RANK', -1)) # https://pytorch.org/docs/stable/elastic/run.html
+RANK = int(os.getenv('RANK', -1))
+WORLD_SIZE = int(os.getenv('WORLD_SIZE', 1))
+GIT_INFO = check_git_info()
+
+
+def train(opt, device):
+ init_seeds(opt.seed + 1 + RANK, deterministic=True)
+ save_dir, data, bs, epochs, nw, imgsz, pretrained = \
+ opt.save_dir, Path(opt.data), opt.batch_size, opt.epochs, min(os.cpu_count() - 1, opt.workers), \
+ opt.imgsz, str(opt.pretrained).lower() == 'true'
+ cuda = device.type != 'cpu'
+
+ # Directories
+ wdir = save_dir / 'weights'
+ wdir.mkdir(parents=True, exist_ok=True) # make dir
+ last, best = wdir / 'last.pt', wdir / 'best.pt'
+
+ # Save run settings
+ yaml_save(save_dir / 'opt.yaml', vars(opt))
+
+ # Logger
+ logger = GenericLogger(opt=opt, console_logger=LOGGER) if RANK in {-1, 0} else None
+
+ # Download Dataset
+ with torch_distributed_zero_first(LOCAL_RANK), WorkingDirectory(ROOT):
+ data_dir = data if data.is_dir() else (DATASETS_DIR / data)
+ if not data_dir.is_dir():
+ LOGGER.info(f'\nDataset not found ⚠️, missing path {data_dir}, attempting download...')
+ t = time.time()
+ if str(data) == 'imagenet':
+ subprocess.run(f"bash {ROOT / 'data/scripts/get_imagenet.sh'}", shell=True, check=True)
+ else:
+ url = f'https://github.com/ultralytics/yolov5/releases/download/v1.0/{data}.zip'
+ download(url, dir=data_dir.parent)
+ s = f"Dataset download success ✅ ({time.time() - t:.1f}s), saved to {colorstr('bold', data_dir)}\n"
+ LOGGER.info(s)
+
+ # Dataloaders
+ nc = len([x for x in (data_dir / 'train').glob('*') if x.is_dir()]) # number of classes
+ trainloader = create_classification_dataloader(path=data_dir / 'train',
+ imgsz=imgsz,
+ batch_size=bs // WORLD_SIZE,
+ augment=True,
+ cache=opt.cache,
+ rank=LOCAL_RANK,
+ workers=nw)
+
+ test_dir = data_dir / 'test' if (data_dir / 'test').exists() else data_dir / 'val' # data/test or data/val
+ if RANK in {-1, 0}:
+ testloader = create_classification_dataloader(path=test_dir,
+ imgsz=imgsz,
+ batch_size=bs // WORLD_SIZE * 2,
+ augment=False,
+ cache=opt.cache,
+ rank=-1,
+ workers=nw)
+
+ # Model
+ with torch_distributed_zero_first(LOCAL_RANK), WorkingDirectory(ROOT):
+ if Path(opt.model).is_file() or opt.model.endswith('.pt'):
+ model = attempt_load(opt.model, device='cpu', fuse=False)
+ elif opt.model in torchvision.models.__dict__: # TorchVision models i.e. resnet50, efficientnet_b0
+ model = torchvision.models.__dict__[opt.model](weights='IMAGENET1K_V1' if pretrained else None)
+ else:
+ m = hub.list('ultralytics/yolov5') # + hub.list('pytorch/vision') # models
+ raise ModuleNotFoundError(f'--model {opt.model} not found. Available models are: \n' + '\n'.join(m))
+ if isinstance(model, DetectionModel):
+ LOGGER.warning("WARNING ⚠️ pass YOLOv5 classifier model with '-cls' suffix, i.e. '--model yolov5s-cls.pt'")
+ model = ClassificationModel(model=model, nc=nc, cutoff=opt.cutoff or 10) # convert to classification model
+ reshape_classifier_output(model, nc) # update class count
+ for m in model.modules():
+ if not pretrained and hasattr(m, 'reset_parameters'):
+ m.reset_parameters()
+ if isinstance(m, torch.nn.Dropout) and opt.dropout is not None:
+ m.p = opt.dropout # set dropout
+ for p in model.parameters():
+ p.requires_grad = True # for training
+ model = model.to(device)
+
+ # Info
+ if RANK in {-1, 0}:
+ model.names = trainloader.dataset.classes # attach class names
+ model.transforms = testloader.dataset.torch_transforms # attach inference transforms
+ model_info(model)
+ if opt.verbose:
+ LOGGER.info(model)
+ images, labels = next(iter(trainloader))
+ file = imshow_cls(images[:25], labels[:25], names=model.names, f=save_dir / 'train_images.jpg')
+ logger.log_images(file, name='Train Examples')
+ logger.log_graph(model, imgsz) # log model
+
+ # Optimizer
+ optimizer = smart_optimizer(model, opt.optimizer, opt.lr0, momentum=0.9, decay=opt.decay)
+
+ # Scheduler
+ lrf = 0.01 # final lr (fraction of lr0)
+ # lf = lambda x: ((1 + math.cos(x * math.pi / epochs)) / 2) * (1 - lrf) + lrf # cosine
+ lf = lambda x: (1 - x / epochs) * (1 - lrf) + lrf # linear
+ scheduler = lr_scheduler.LambdaLR(optimizer, lr_lambda=lf)
+ # scheduler = lr_scheduler.OneCycleLR(optimizer, max_lr=lr0, total_steps=epochs, pct_start=0.1,
+ # final_div_factor=1 / 25 / lrf)
+
+ # EMA
+ ema = ModelEMA(model) if RANK in {-1, 0} else None
+
+ # DDP mode
+ if cuda and RANK != -1:
+ model = smart_DDP(model)
+
+ # Train
+ t0 = time.time()
+ criterion = smartCrossEntropyLoss(label_smoothing=opt.label_smoothing) # loss function
+ best_fitness = 0.0
+ scaler = amp.GradScaler(enabled=cuda)
+ val = test_dir.stem # 'val' or 'test'
+ LOGGER.info(f'Image sizes {imgsz} train, {imgsz} test\n'
+ f'Using {nw * WORLD_SIZE} dataloader workers\n'
+ f"Logging results to {colorstr('bold', save_dir)}\n"
+ f'Starting {opt.model} training on {data} dataset with {nc} classes for {epochs} epochs...\n\n'
+ f"{'Epoch':>10}{'GPU_mem':>10}{'train_loss':>12}{f'{val}_loss':>12}{'top1_acc':>12}{'top5_acc':>12}")
+ for epoch in range(epochs): # loop over the dataset multiple times
+ tloss, vloss, fitness = 0.0, 0.0, 0.0 # train loss, val loss, fitness
+ model.train()
+ if RANK != -1:
+ trainloader.sampler.set_epoch(epoch)
+ pbar = enumerate(trainloader)
+ if RANK in {-1, 0}:
+ pbar = tqdm(enumerate(trainloader), total=len(trainloader), bar_format=TQDM_BAR_FORMAT)
+ for i, (images, labels) in pbar: # progress bar
+ images, labels = images.to(device, non_blocking=True), labels.to(device)
+
+ # Forward
+ with amp.autocast(enabled=cuda): # stability issues when enabled
+ loss = criterion(model(images), labels)
+
+ # Backward
+ scaler.scale(loss).backward()
+
+ # Optimize
+ scaler.unscale_(optimizer) # unscale gradients
+ torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=10.0) # clip gradients
+ scaler.step(optimizer)
+ scaler.update()
+ optimizer.zero_grad()
+ if ema:
+ ema.update(model)
+
+ if RANK in {-1, 0}:
+ # Print
+ tloss = (tloss * i + loss.item()) / (i + 1) # update mean losses
+ mem = '%.3gG' % (torch.cuda.memory_reserved() / 1E9 if torch.cuda.is_available() else 0) # (GB)
+ pbar.desc = f"{f'{epoch + 1}/{epochs}':>10}{mem:>10}{tloss:>12.3g}" + ' ' * 36
+
+ # Test
+ if i == len(pbar) - 1: # last batch
+ top1, top5, vloss = validate.run(model=ema.ema,
+ dataloader=testloader,
+ criterion=criterion,
+ pbar=pbar) # test accuracy, loss
+ fitness = top1 # define fitness as top1 accuracy
+
+ # Scheduler
+ scheduler.step()
+
+ # Log metrics
+ if RANK in {-1, 0}:
+ # Best fitness
+ if fitness > best_fitness:
+ best_fitness = fitness
+
+ # Log
+ metrics = {
+ "train/loss": tloss,
+ f"{val}/loss": vloss,
+ "metrics/accuracy_top1": top1,
+ "metrics/accuracy_top5": top5,
+ "lr/0": optimizer.param_groups[0]['lr']} # learning rate
+ logger.log_metrics(metrics, epoch)
+
+ # Save model
+ final_epoch = epoch + 1 == epochs
+ if (not opt.nosave) or final_epoch:
+ ckpt = {
+ 'epoch': epoch,
+ 'best_fitness': best_fitness,
+ 'model': deepcopy(ema.ema).half(), # deepcopy(de_parallel(model)).half(),
+ 'ema': None, # deepcopy(ema.ema).half(),
+ 'updates': ema.updates,
+ 'optimizer': None, # optimizer.state_dict(),
+ 'opt': vars(opt),
+ 'git': GIT_INFO, # {remote, branch, commit} if a git repo
+ 'date': datetime.now().isoformat()}
+
+ # Save last, best and delete
+ torch.save(ckpt, last)
+ if best_fitness == fitness:
+ torch.save(ckpt, best)
+ del ckpt
+
+ # Train complete
+ if RANK in {-1, 0} and final_epoch:
+ LOGGER.info(f'\nTraining complete ({(time.time() - t0) / 3600:.3f} hours)'
+ f"\nResults saved to {colorstr('bold', save_dir)}"
+ f"\nPredict: python classify/predict.py --weights {best} --source im.jpg"
+ f"\nValidate: python classify/val.py --weights {best} --data {data_dir}"
+ f"\nExport: python export.py --weights {best} --include onnx"
+ f"\nPyTorch Hub: model = torch.hub.load('ultralytics/yolov5', 'custom', '{best}')"
+ f"\nVisualize: https://netron.app\n")
+
+ # Plot examples
+ images, labels = (x[:25] for x in next(iter(testloader))) # first 25 images and labels
+ pred = torch.max(ema.ema(images.to(device)), 1)[1]
+ file = imshow_cls(images, labels, pred, model.names, verbose=False, f=save_dir / 'test_images.jpg')
+
+ # Log results
+ meta = {"epochs": epochs, "top1_acc": best_fitness, "date": datetime.now().isoformat()}
+ logger.log_images(file, name='Test Examples (true-predicted)', epoch=epoch)
+ logger.log_model(best, epochs, metadata=meta)
+
+
+def parse_opt(known=False):
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--model', type=str, default='yolov5s-cls.pt', help='initial weights path')
+ parser.add_argument('--data', type=str, default='imagenette160', help='cifar10, cifar100, mnist, imagenet, ...')
+ parser.add_argument('--epochs', type=int, default=10, help='total training epochs')
+ parser.add_argument('--batch-size', type=int, default=64, help='total batch size for all GPUs')
+ parser.add_argument('--imgsz', '--img', '--img-size', type=int, default=224, help='train, val image size (pixels)')
+ parser.add_argument('--nosave', action='store_true', help='only save final checkpoint')
+ parser.add_argument('--cache', type=str, nargs='?', const='ram', help='--cache images in "ram" (default) or "disk"')
+ parser.add_argument('--device', default='', help='cuda device, i.e. 0 or 0,1,2,3 or cpu')
+ parser.add_argument('--workers', type=int, default=8, help='max dataloader workers (per RANK in DDP mode)')
+ parser.add_argument('--project', default=ROOT / 'runs/train-cls', help='save to project/name')
+ parser.add_argument('--name', default='exp', help='save to project/name')
+ parser.add_argument('--exist-ok', action='store_true', help='existing project/name ok, do not increment')
+ parser.add_argument('--pretrained', nargs='?', const=True, default=True, help='start from i.e. --pretrained False')
+ parser.add_argument('--optimizer', choices=['SGD', 'Adam', 'AdamW', 'RMSProp'], default='Adam', help='optimizer')
+ parser.add_argument('--lr0', type=float, default=0.001, help='initial learning rate')
+ parser.add_argument('--decay', type=float, default=5e-5, help='weight decay')
+ parser.add_argument('--label-smoothing', type=float, default=0.1, help='Label smoothing epsilon')
+ parser.add_argument('--cutoff', type=int, default=None, help='Model layer cutoff index for Classify() head')
+ parser.add_argument('--dropout', type=float, default=None, help='Dropout (fraction)')
+ parser.add_argument('--verbose', action='store_true', help='Verbose mode')
+ parser.add_argument('--seed', type=int, default=0, help='Global training seed')
+ parser.add_argument('--local_rank', type=int, default=-1, help='Automatic DDP Multi-GPU argument, do not modify')
+ return parser.parse_known_args()[0] if known else parser.parse_args()
+
+
+def main(opt):
+ # Checks
+ if RANK in {-1, 0}:
+ print_args(vars(opt))
+ check_git_status()
+ check_requirements()
+
+ # DDP mode
+ device = select_device(opt.device, batch_size=opt.batch_size)
+ if LOCAL_RANK != -1:
+ assert opt.batch_size != -1, 'AutoBatch is coming soon for classification, please pass a valid --batch-size'
+ assert opt.batch_size % WORLD_SIZE == 0, f'--batch-size {opt.batch_size} must be multiple of WORLD_SIZE'
+ assert torch.cuda.device_count() > LOCAL_RANK, 'insufficient CUDA devices for DDP command'
+ torch.cuda.set_device(LOCAL_RANK)
+ device = torch.device('cuda', LOCAL_RANK)
+ dist.init_process_group(backend="nccl" if dist.is_nccl_available() else "gloo")
+
+ # Parameters
+ opt.save_dir = increment_path(Path(opt.project) / opt.name, exist_ok=opt.exist_ok) # increment run
+
+ # Train
+ train(opt, device)
+
+
+def run(**kwargs):
+ # Usage: from yolov5 import classify; classify.train.run(data=mnist, imgsz=320, model='yolov5m')
+ opt = parse_opt(True)
+ for k, v in kwargs.items():
+ setattr(opt, k, v)
+ main(opt)
+ return opt
+
+
+if __name__ == "__main__":
+ opt = parse_opt()
+ main(opt)
diff --git a/classify/val.py b/classify/val.py
new file mode 100644
index 0000000000000000000000000000000000000000..8657036fb2a23d7388240c31d36b67b95877ec12
--- /dev/null
+++ b/classify/val.py
@@ -0,0 +1,170 @@
+# YOLOv5 🚀 by Ultralytics, GPL-3.0 license
+"""
+Validate a trained YOLOv5 classification model on a classification dataset
+
+Usage:
+ $ bash data/scripts/get_imagenet.sh --val # download ImageNet val split (6.3G, 50000 images)
+ $ python classify/val.py --weights yolov5m-cls.pt --data ../datasets/imagenet --img 224 # validate ImageNet
+
+Usage - formats:
+ $ python classify/val.py --weights yolov5s-cls.pt # PyTorch
+ yolov5s-cls.torchscript # TorchScript
+ yolov5s-cls.onnx # ONNX Runtime or OpenCV DNN with --dnn
+ yolov5s-cls_openvino_model # OpenVINO
+ yolov5s-cls.engine # TensorRT
+ yolov5s-cls.mlmodel # CoreML (macOS-only)
+ yolov5s-cls_saved_model # TensorFlow SavedModel
+ yolov5s-cls.pb # TensorFlow GraphDef
+ yolov5s-cls.tflite # TensorFlow Lite
+ yolov5s-cls_edgetpu.tflite # TensorFlow Edge TPU
+ yolov5s-cls_paddle_model # PaddlePaddle
+"""
+
+import argparse
+import os
+import sys
+from pathlib import Path
+
+import torch
+from tqdm import tqdm
+
+FILE = Path(__file__).resolve()
+ROOT = FILE.parents[1] # YOLOv5 root directory
+if str(ROOT) not in sys.path:
+ sys.path.append(str(ROOT)) # add ROOT to PATH
+ROOT = Path(os.path.relpath(ROOT, Path.cwd())) # relative
+
+from models.common import DetectMultiBackend
+from utils.dataloaders import create_classification_dataloader
+from utils.general import (LOGGER, TQDM_BAR_FORMAT, Profile, check_img_size, check_requirements, colorstr,
+ increment_path, print_args)
+from utils.torch_utils import select_device, smart_inference_mode
+
+
+@smart_inference_mode()
+def run(
+ data=ROOT / '../datasets/mnist', # dataset dir
+ weights=ROOT / 'yolov5s-cls.pt', # model.pt path(s)
+ batch_size=128, # batch size
+ imgsz=224, # inference size (pixels)
+ device='', # cuda device, i.e. 0 or 0,1,2,3 or cpu
+ workers=8, # max dataloader workers (per RANK in DDP mode)
+ verbose=False, # verbose output
+ project=ROOT / 'runs/val-cls', # save to project/name
+ name='exp', # save to project/name
+ exist_ok=False, # existing project/name ok, do not increment
+ half=False, # use FP16 half-precision inference
+ dnn=False, # use OpenCV DNN for ONNX inference
+ model=None,
+ dataloader=None,
+ criterion=None,
+ pbar=None,
+):
+ # Initialize/load model and set device
+ training = model is not None
+ if training: # called by train.py
+ device, pt, jit, engine = next(model.parameters()).device, True, False, False # get model device, PyTorch model
+ half &= device.type != 'cpu' # half precision only supported on CUDA
+ model.half() if half else model.float()
+ else: # called directly
+ device = select_device(device, batch_size=batch_size)
+
+ # Directories
+ save_dir = increment_path(Path(project) / name, exist_ok=exist_ok) # increment run
+ save_dir.mkdir(parents=True, exist_ok=True) # make dir
+
+ # Load model
+ model = DetectMultiBackend(weights, device=device, dnn=dnn, fp16=half)
+ stride, pt, jit, engine = model.stride, model.pt, model.jit, model.engine
+ imgsz = check_img_size(imgsz, s=stride) # check image size
+ half = model.fp16 # FP16 supported on limited backends with CUDA
+ if engine:
+ batch_size = model.batch_size
+ else:
+ device = model.device
+ if not (pt or jit):
+ batch_size = 1 # export.py models default to batch-size 1
+ LOGGER.info(f'Forcing --batch-size 1 square inference (1,3,{imgsz},{imgsz}) for non-PyTorch models')
+
+ # Dataloader
+ data = Path(data)
+ test_dir = data / 'test' if (data / 'test').exists() else data / 'val' # data/test or data/val
+ dataloader = create_classification_dataloader(path=test_dir,
+ imgsz=imgsz,
+ batch_size=batch_size,
+ augment=False,
+ rank=-1,
+ workers=workers)
+
+ model.eval()
+ pred, targets, loss, dt = [], [], 0, (Profile(), Profile(), Profile())
+ n = len(dataloader) # number of batches
+ action = 'validating' if dataloader.dataset.root.stem == 'val' else 'testing'
+ desc = f"{pbar.desc[:-36]}{action:>36}" if pbar else f"{action}"
+ bar = tqdm(dataloader, desc, n, not training, bar_format=TQDM_BAR_FORMAT, position=0)
+ with torch.cuda.amp.autocast(enabled=device.type != 'cpu'):
+ for images, labels in bar:
+ with dt[0]:
+ images, labels = images.to(device, non_blocking=True), labels.to(device)
+
+ with dt[1]:
+ y = model(images)
+
+ with dt[2]:
+ pred.append(y.argsort(1, descending=True)[:, :5])
+ targets.append(labels)
+ if criterion:
+ loss += criterion(y, labels)
+
+ loss /= n
+ pred, targets = torch.cat(pred), torch.cat(targets)
+ correct = (targets[:, None] == pred).float()
+ acc = torch.stack((correct[:, 0], correct.max(1).values), dim=1) # (top1, top5) accuracy
+ top1, top5 = acc.mean(0).tolist()
+
+ if pbar:
+ pbar.desc = f"{pbar.desc[:-36]}{loss:>12.3g}{top1:>12.3g}{top5:>12.3g}"
+ if verbose: # all classes
+ LOGGER.info(f"{'Class':>24}{'Images':>12}{'top1_acc':>12}{'top5_acc':>12}")
+ LOGGER.info(f"{'all':>24}{targets.shape[0]:>12}{top1:>12.3g}{top5:>12.3g}")
+ for i, c in model.names.items():
+ aci = acc[targets == i]
+ top1i, top5i = aci.mean(0).tolist()
+ LOGGER.info(f"{c:>24}{aci.shape[0]:>12}{top1i:>12.3g}{top5i:>12.3g}")
+
+ # Print results
+ t = tuple(x.t / len(dataloader.dataset.samples) * 1E3 for x in dt) # speeds per image
+ shape = (1, 3, imgsz, imgsz)
+ LOGGER.info(f'Speed: %.1fms pre-process, %.1fms inference, %.1fms post-process per image at shape {shape}' % t)
+ LOGGER.info(f"Results saved to {colorstr('bold', save_dir)}")
+
+ return top1, top5, loss
+
+
+def parse_opt():
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--data', type=str, default=ROOT / '../datasets/mnist', help='dataset path')
+ parser.add_argument('--weights', nargs='+', type=str, default=ROOT / 'yolov5s-cls.pt', help='model.pt path(s)')
+ parser.add_argument('--batch-size', type=int, default=128, help='batch size')
+ parser.add_argument('--imgsz', '--img', '--img-size', type=int, default=224, help='inference size (pixels)')
+ parser.add_argument('--device', default='', help='cuda device, i.e. 0 or 0,1,2,3 or cpu')
+ parser.add_argument('--workers', type=int, default=8, help='max dataloader workers (per RANK in DDP mode)')
+ parser.add_argument('--verbose', nargs='?', const=True, default=True, help='verbose output')
+ parser.add_argument('--project', default=ROOT / 'runs/val-cls', help='save to project/name')
+ parser.add_argument('--name', default='exp', help='save to project/name')
+ parser.add_argument('--exist-ok', action='store_true', help='existing project/name ok, do not increment')
+ parser.add_argument('--half', action='store_true', help='use FP16 half-precision inference')
+ parser.add_argument('--dnn', action='store_true', help='use OpenCV DNN for ONNX inference')
+ opt = parser.parse_args()
+ print_args(vars(opt))
+ return opt
+
+
+def main(opt):
+ check_requirements(exclude=('tensorboard', 'thop'))
+ run(**vars(opt))
+
+
+if __name__ == "__main__":
+ opt = parse_opt()
+ main(opt)
diff --git a/data/coco.yaml b/data/coco.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..9276e76b28d413aad6a1107f8b4a0edfe6594fa0
--- /dev/null
+++ b/data/coco.yaml
@@ -0,0 +1,125 @@
+path: ../datasets/coco # dataset root dir
+train: train2017.txt # train images (relative to 'path') 118287 images
+val: val2017.txt # val images (relative to 'path') 5000 images
+test: test-dev2017.txt # 20288 of 40670 images, submit to https://competitions.codalab.org/competitions/20794
+
+# Classes
+names:
+ 0: person
+ 1: bicycle
+ 2: car
+ 3: motorcycle
+ 4: airplane
+ 5: bus
+ 6: train
+ 7: truck
+ 8: boat
+ 9: traffic light
+ 10: fire hydrant
+ 11: stop sign
+ 12: parking meter
+ 13: bench
+ 14: bird
+ 15: cat
+ 16: dog
+ 17: horse
+ 18: sheep
+ 19: cow
+ 20: elephant
+ 21: bear
+ 22: zebra
+ 23: giraffe
+ 24: backpack
+ 25: umbrella
+ 26: handbag
+ 27: tie
+ 28: suitcase
+ 29: frisbee
+ 30: skis
+ 31: snowboard
+ 32: sports ball
+ 33: kite
+ 34: baseball bat
+ 35: baseball glove
+ 36: skateboard
+ 37: surfboard
+ 38: tennis racket
+ 39: bottle
+ 40: wine glass
+ 41: cup
+ 42: fork
+ 43: knife
+ 44: spoon
+ 45: bowl
+ 46: banana
+ 47: apple
+ 48: sandwich
+ 49: orange
+ 50: broccoli
+ 51: carrot
+ 52: hot dog
+ 53: pizza
+ 54: donut
+ 55: cake
+ 56: chair
+ 57: couch
+ 58: potted plant
+ 59: bed
+ 60: dining table
+ 61: toilet
+ 62: tv
+ 63: laptop
+ 64: mouse
+ 65: remote
+ 66: keyboard
+ 67: cell phone
+ 68: microwave
+ 69: oven
+ 70: toaster
+ 71: sink
+ 72: refrigerator
+ 73: book
+ 74: clock
+ 75: vase
+ 76: scissors
+ 77: teddy bear
+ 78: hair drier
+ 79: toothbrush
+
+
+# stuff names
+stuff_names: [
+ 'banner', 'blanket', 'branch', 'bridge', 'building-other', 'bush', 'cabinet', 'cage',
+ 'cardboard', 'carpet', 'ceiling-other', 'ceiling-tile', 'cloth', 'clothes', 'clouds', 'counter', 'cupboard',
+ 'curtain', 'desk-stuff', 'dirt', 'door-stuff', 'fence', 'floor-marble', 'floor-other', 'floor-stone', 'floor-tile',
+ 'floor-wood', 'flower', 'fog', 'food-other', 'fruit', 'furniture-other', 'grass', 'gravel', 'ground-other', 'hill',
+ 'house', 'leaves', 'light', 'mat', 'metal', 'mirror-stuff', 'moss', 'mountain', 'mud', 'napkin', 'net', 'paper',
+ 'pavement', 'pillow', 'plant-other', 'plastic', 'platform', 'playingfield', 'railing', 'railroad', 'river', 'road',
+ 'rock', 'roof', 'rug', 'salad', 'sand', 'sea', 'shelf', 'sky-other', 'skyscraper', 'snow', 'solid-other', 'stairs',
+ 'stone', 'straw', 'structural-other', 'table', 'tent', 'textile-other', 'towel', 'tree', 'vegetable', 'wall-brick',
+ 'wall-concrete', 'wall-other', 'wall-panel', 'wall-stone', 'wall-tile', 'wall-wood', 'water-other', 'waterdrops',
+ 'window-blind', 'window-other', 'wood',
+ # other
+ 'other',
+ # unlabeled
+ 'unlabeled'
+]
+
+
+# Download script/URL (optional)
+download: |
+ from utils.general import download, Path
+
+
+ # Download labels
+ #segments = True # segment or box labels
+ #dir = Path(yaml['path']) # dataset root dir
+ #url = 'https://github.com/WongKinYiu/yolov7/releases/download/v0.1/'
+ #urls = [url + ('coco2017labels-segments.zip' if segments else 'coco2017labels.zip')] # labels
+ #download(urls, dir=dir.parent)
+
+ # Download data
+ #urls = ['http://images.cocodataset.org/zips/train2017.zip', # 19G, 118k images
+ # 'http://images.cocodataset.org/zips/val2017.zip', # 1G, 5k images
+ # 'http://images.cocodataset.org/zips/test2017.zip'] # 7G, 41k images (optional)
+ #download(urls, dir=dir / 'images', threads=3)
diff --git a/data/hyps/hyp.scratch-high.yaml b/data/hyps/hyp.scratch-high.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..fdb2c378800d57862827961494e019e44f63a59c
--- /dev/null
+++ b/data/hyps/hyp.scratch-high.yaml
@@ -0,0 +1,30 @@
+lr0: 0.01 # initial learning rate (SGD=1E-2, Adam=1E-3)
+lrf: 0.01 # final OneCycleLR learning rate (lr0 * lrf)
+momentum: 0.937 # SGD momentum/Adam beta1
+weight_decay: 0.0005 # optimizer weight decay 5e-4
+warmup_epochs: 3.0 # warmup epochs (fractions ok)
+warmup_momentum: 0.8 # warmup initial momentum
+warmup_bias_lr: 0.1 # warmup initial bias lr
+box: 7.5 # box loss gain
+cls: 0.5 # cls loss gain
+cls_pw: 1.0 # cls BCELoss positive_weight
+obj: 0.7 # obj loss gain (scale with pixels)
+obj_pw: 1.0 # obj BCELoss positive_weight
+dfl: 1.5 # dfl loss gain
+iou_t: 0.20 # IoU training threshold
+anchor_t: 5.0 # anchor-multiple threshold
+# anchors: 3 # anchors per output layer (0 to ignore)
+fl_gamma: 0.0 # focal loss gamma (efficientDet default gamma=1.5)
+hsv_h: 0.015 # image HSV-Hue augmentation (fraction)
+hsv_s: 0.7 # image HSV-Saturation augmentation (fraction)
+hsv_v: 0.4 # image HSV-Value augmentation (fraction)
+degrees: 0.0 # image rotation (+/- deg)
+translate: 0.1 # image translation (+/- fraction)
+scale: 0.9 # image scale (+/- gain)
+shear: 0.0 # image shear (+/- deg)
+perspective: 0.0 # image perspective (+/- fraction), range 0-0.001
+flipud: 0.0 # image flip up-down (probability)
+fliplr: 0.5 # image flip left-right (probability)
+mosaic: 1.0 # image mosaic (probability)
+mixup: 0.15 # image mixup (probability)
+copy_paste: 0.3 # segment copy-paste (probability)
diff --git a/data/images/horses.jpg b/data/images/horses.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..3a761f46ba08ed459af026b59f6b91b6fa597dd1
Binary files /dev/null and b/data/images/horses.jpg differ
diff --git a/detect.py b/detect.py
new file mode 100644
index 0000000000000000000000000000000000000000..6dbb6e7ef54e862023ec45cb2abb0333124358f7
--- /dev/null
+++ b/detect.py
@@ -0,0 +1,231 @@
+import argparse
+import os
+import platform
+import sys
+from pathlib import Path
+
+import torch
+
+FILE = Path(__file__).resolve()
+ROOT = FILE.parents[0] # YOLO root directory
+if str(ROOT) not in sys.path:
+ sys.path.append(str(ROOT)) # add ROOT to PATH
+ROOT = Path(os.path.relpath(ROOT, Path.cwd())) # relative
+
+from models.common import DetectMultiBackend
+from utils.dataloaders import IMG_FORMATS, VID_FORMATS, LoadImages, LoadScreenshots, LoadStreams
+from utils.general import (LOGGER, Profile, check_file, check_img_size, check_imshow, check_requirements, colorstr, cv2,
+ increment_path, non_max_suppression, print_args, scale_boxes, strip_optimizer, xyxy2xywh)
+from utils.plots import Annotator, colors, save_one_box
+from utils.torch_utils import select_device, smart_inference_mode
+
+
+@smart_inference_mode()
+def run(
+ weights=ROOT / 'yolo.pt', # model path or triton URL
+ source=ROOT / 'data/images', # file/dir/URL/glob/screen/0(webcam)
+ data=ROOT / 'data/coco.yaml', # dataset.yaml path
+ imgsz=(640, 640), # inference size (height, width)
+ conf_thres=0.25, # confidence threshold
+ iou_thres=0.45, # NMS IOU threshold
+ max_det=1000, # maximum detections per image
+ device='', # cuda device, i.e. 0 or 0,1,2,3 or cpu
+ view_img=False, # show results
+ save_txt=False, # save results to *.txt
+ save_conf=False, # save confidences in --save-txt labels
+ save_crop=False, # save cropped prediction boxes
+ nosave=False, # do not save images/videos
+ classes=None, # filter by class: --class 0, or --class 0 2 3
+ agnostic_nms=False, # class-agnostic NMS
+ augment=False, # augmented inference
+ visualize=False, # visualize features
+ update=False, # update all models
+ project=ROOT / 'runs/detect', # save results to project/name
+ name='exp', # save results to project/name
+ exist_ok=False, # existing project/name ok, do not increment
+ line_thickness=3, # bounding box thickness (pixels)
+ hide_labels=False, # hide labels
+ hide_conf=False, # hide confidences
+ half=False, # use FP16 half-precision inference
+ dnn=False, # use OpenCV DNN for ONNX inference
+ vid_stride=1, # video frame-rate stride
+):
+ source = str(source)
+ save_img = not nosave and not source.endswith('.txt') # save inference images
+ is_file = Path(source).suffix[1:] in (IMG_FORMATS + VID_FORMATS)
+ is_url = source.lower().startswith(('rtsp://', 'rtmp://', 'http://', 'https://'))
+ webcam = source.isnumeric() or source.endswith('.txt') or (is_url and not is_file)
+ screenshot = source.lower().startswith('screen')
+ if is_url and is_file:
+ source = check_file(source) # download
+
+ # Directories
+ save_dir = increment_path(Path(project) / name, exist_ok=exist_ok) # increment run
+ (save_dir / 'labels' if save_txt else save_dir).mkdir(parents=True, exist_ok=True) # make dir
+
+ # Load model
+ device = select_device(device)
+ model = DetectMultiBackend(weights, device=device, dnn=dnn, data=data, fp16=half)
+ stride, names, pt = model.stride, model.names, model.pt
+ imgsz = check_img_size(imgsz, s=stride) # check image size
+
+ # Dataloader
+ bs = 1 # batch_size
+ if webcam:
+ view_img = check_imshow(warn=True)
+ dataset = LoadStreams(source, img_size=imgsz, stride=stride, auto=pt, vid_stride=vid_stride)
+ bs = len(dataset)
+ elif screenshot:
+ dataset = LoadScreenshots(source, img_size=imgsz, stride=stride, auto=pt)
+ else:
+ dataset = LoadImages(source, img_size=imgsz, stride=stride, auto=pt, vid_stride=vid_stride)
+ vid_path, vid_writer = [None] * bs, [None] * bs
+
+ # Run inference
+ model.warmup(imgsz=(1 if pt or model.triton else bs, 3, *imgsz)) # warmup
+ seen, windows, dt = 0, [], (Profile(), Profile(), Profile())
+ for path, im, im0s, vid_cap, s in dataset:
+ with dt[0]:
+ im = torch.from_numpy(im).to(model.device)
+ im = im.half() if model.fp16 else im.float() # uint8 to fp16/32
+ im /= 255 # 0 - 255 to 0.0 - 1.0
+ if len(im.shape) == 3:
+ im = im[None] # expand for batch dim
+
+ # Inference
+ with dt[1]:
+ visualize = increment_path(save_dir / Path(path).stem, mkdir=True) if visualize else False
+ pred = model(im, augment=augment, visualize=visualize)
+
+ # NMS
+ with dt[2]:
+ pred = non_max_suppression(pred, conf_thres, iou_thres, classes, agnostic_nms, max_det=max_det)
+
+ # Second-stage classifier (optional)
+ # pred = utils.general.apply_classifier(pred, classifier_model, im, im0s)
+
+ # Process predictions
+ for i, det in enumerate(pred): # per image
+ seen += 1
+ if webcam: # batch_size >= 1
+ p, im0, frame = path[i], im0s[i].copy(), dataset.count
+ s += f'{i}: '
+ else:
+ p, im0, frame = path, im0s.copy(), getattr(dataset, 'frame', 0)
+
+ p = Path(p) # to Path
+ save_path = str(save_dir / p.name) # im.jpg
+ txt_path = str(save_dir / 'labels' / p.stem) + ('' if dataset.mode == 'image' else f'_{frame}') # im.txt
+ s += '%gx%g ' % im.shape[2:] # print string
+ gn = torch.tensor(im0.shape)[[1, 0, 1, 0]] # normalization gain whwh
+ imc = im0.copy() if save_crop else im0 # for save_crop
+ annotator = Annotator(im0, line_width=line_thickness, example=str(names))
+ if len(det):
+ # Rescale boxes from img_size to im0 size
+ det[:, :4] = scale_boxes(im.shape[2:], det[:, :4], im0.shape).round()
+
+ # Print results
+ for c in det[:, 5].unique():
+ n = (det[:, 5] == c).sum() # detections per class
+ s += f"{n} {names[int(c)]}{'s' * (n > 1)}, " # add to string
+
+ # Write results
+ for *xyxy, conf, cls in reversed(det):
+ if save_txt: # Write to file
+ xywh = (xyxy2xywh(torch.tensor(xyxy).view(1, 4)) / gn).view(-1).tolist() # normalized xywh
+ line = (cls, *xywh, conf) if save_conf else (cls, *xywh) # label format
+ with open(f'{txt_path}.txt', 'a') as f:
+ f.write(('%g ' * len(line)).rstrip() % line + '\n')
+
+ if save_img or save_crop or view_img: # Add bbox to image
+ c = int(cls) # integer class
+ label = None if hide_labels else (names[c] if hide_conf else f'{names[c]} {conf:.2f}')
+ annotator.box_label(xyxy, label, color=colors(c, True))
+ if save_crop:
+ save_one_box(xyxy, imc, file=save_dir / 'crops' / names[c] / f'{p.stem}.jpg', BGR=True)
+
+ # Stream results
+ im0 = annotator.result()
+ if view_img:
+ if platform.system() == 'Linux' and p not in windows:
+ windows.append(p)
+ cv2.namedWindow(str(p), cv2.WINDOW_NORMAL | cv2.WINDOW_KEEPRATIO) # allow window resize (Linux)
+ cv2.resizeWindow(str(p), im0.shape[1], im0.shape[0])
+ cv2.imshow(str(p), im0)
+ cv2.waitKey(1) # 1 millisecond
+
+ # Save results (image with detections)
+ if save_img:
+ if dataset.mode == 'image':
+ cv2.imwrite(save_path, im0)
+ else: # 'video' or 'stream'
+ if vid_path[i] != save_path: # new video
+ vid_path[i] = save_path
+ if isinstance(vid_writer[i], cv2.VideoWriter):
+ vid_writer[i].release() # release previous video writer
+ if vid_cap: # video
+ fps = vid_cap.get(cv2.CAP_PROP_FPS)
+ w = int(vid_cap.get(cv2.CAP_PROP_FRAME_WIDTH))
+ h = int(vid_cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
+ else: # stream
+ fps, w, h = 30, im0.shape[1], im0.shape[0]
+ save_path = str(Path(save_path).with_suffix('.mp4')) # force *.mp4 suffix on results videos
+ vid_writer[i] = cv2.VideoWriter(save_path, cv2.VideoWriter_fourcc(*'mp4v'), fps, (w, h))
+ vid_writer[i].write(im0)
+
+ # Print time (inference-only)
+ LOGGER.info(f"{s}{'' if len(det) else '(no detections), '}{dt[1].dt * 1E3:.1f}ms")
+
+ # Print results
+ t = tuple(x.t / seen * 1E3 for x in dt) # speeds per image
+ LOGGER.info(f'Speed: %.1fms pre-process, %.1fms inference, %.1fms NMS per image at shape {(1, 3, *imgsz)}' % t)
+ if save_txt or save_img:
+ s = f"\n{len(list(save_dir.glob('labels/*.txt')))} labels saved to {save_dir / 'labels'}" if save_txt else ''
+ LOGGER.info(f"Results saved to {colorstr('bold', save_dir)}{s}")
+ if update:
+ strip_optimizer(weights[0]) # update model (to fix SourceChangeWarning)
+
+
+def parse_opt():
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--weights', nargs='+', type=str, default=ROOT / 'yolo.pt', help='model path or triton URL')
+ parser.add_argument('--source', type=str, default=ROOT / 'data/images', help='file/dir/URL/glob/screen/0(webcam)')
+ parser.add_argument('--data', type=str, default=ROOT / 'data/coco128.yaml', help='(optional) dataset.yaml path')
+ parser.add_argument('--imgsz', '--img', '--img-size', nargs='+', type=int, default=[640], help='inference size h,w')
+ parser.add_argument('--conf-thres', type=float, default=0.25, help='confidence threshold')
+ parser.add_argument('--iou-thres', type=float, default=0.45, help='NMS IoU threshold')
+ parser.add_argument('--max-det', type=int, default=1000, help='maximum detections per image')
+ parser.add_argument('--device', default='', help='cuda device, i.e. 0 or 0,1,2,3 or cpu')
+ parser.add_argument('--view-img', action='store_true', help='show results')
+ parser.add_argument('--save-txt', action='store_true', help='save results to *.txt')
+ parser.add_argument('--save-conf', action='store_true', help='save confidences in --save-txt labels')
+ parser.add_argument('--save-crop', action='store_true', help='save cropped prediction boxes')
+ parser.add_argument('--nosave', action='store_true', help='do not save images/videos')
+ parser.add_argument('--classes', nargs='+', type=int, help='filter by class: --classes 0, or --classes 0 2 3')
+ parser.add_argument('--agnostic-nms', action='store_true', help='class-agnostic NMS')
+ parser.add_argument('--augment', action='store_true', help='augmented inference')
+ parser.add_argument('--visualize', action='store_true', help='visualize features')
+ parser.add_argument('--update', action='store_true', help='update all models')
+ parser.add_argument('--project', default=ROOT / 'runs/detect', help='save results to project/name')
+ parser.add_argument('--name', default='exp', help='save results to project/name')
+ parser.add_argument('--exist-ok', action='store_true', help='existing project/name ok, do not increment')
+ parser.add_argument('--line-thickness', default=3, type=int, help='bounding box thickness (pixels)')
+ parser.add_argument('--hide-labels', default=False, action='store_true', help='hide labels')
+ parser.add_argument('--hide-conf', default=False, action='store_true', help='hide confidences')
+ parser.add_argument('--half', action='store_true', help='use FP16 half-precision inference')
+ parser.add_argument('--dnn', action='store_true', help='use OpenCV DNN for ONNX inference')
+ parser.add_argument('--vid-stride', type=int, default=1, help='video frame-rate stride')
+ opt = parser.parse_args()
+ opt.imgsz *= 2 if len(opt.imgsz) == 1 else 1 # expand
+ print_args(vars(opt))
+ return opt
+
+
+def main(opt):
+ # check_requirements(exclude=('tensorboard', 'thop'))
+ run(**vars(opt))
+
+
+if __name__ == "__main__":
+ opt = parse_opt()
+ main(opt)
diff --git a/detect_dual.py b/detect_dual.py
new file mode 100644
index 0000000000000000000000000000000000000000..f7ce545a35cebd7397f803bee3a5f65b0f1ec969
--- /dev/null
+++ b/detect_dual.py
@@ -0,0 +1,233 @@
+import argparse
+import os
+import platform
+import sys
+from pathlib import Path
+
+
+import torch
+
+FILE = Path(__file__).resolve()
+ROOT = FILE.parents[0] # YOLO root directory
+if str(ROOT) not in sys.path:
+ sys.path.append(str(ROOT)) # add ROOT to PATH
+ROOT = Path(os.path.relpath(ROOT, Path.cwd())) # relative
+
+from models.common import DetectMultiBackend
+from utils.dataloaders import IMG_FORMATS, VID_FORMATS, LoadImages, LoadScreenshots, LoadStreams
+from utils.general import (LOGGER, Profile, check_file, check_img_size, check_imshow, check_requirements, colorstr, cv2,
+ increment_path, non_max_suppression, print_args, scale_boxes, strip_optimizer, xyxy2xywh)
+from utils.plots import Annotator, colors, save_one_box
+from utils.torch_utils import select_device, smart_inference_mode
+
+
+@smart_inference_mode()
+def run(
+ weights=ROOT / 'yolo.pt', # model path or triton URL
+ source=ROOT / 'data/images', # file/dir/URL/glob/screen/0(webcam)
+ data=ROOT / 'data/coco.yaml', # dataset.yaml path
+ imgsz=(640, 640), # inference size (height, width)
+ conf_thres=0.25, # confidence threshold
+ iou_thres=0.45, # NMS IOU threshold
+ max_det=1000, # maximum detections per image
+ device='', # cuda device, i.e. 0 or 0,1,2,3 or cpu
+ view_img=False, # show results
+ save_txt=False, # save results to *.txt
+ save_conf=False, # save confidences in --save-txt labels
+ save_crop=False, # save cropped prediction boxes
+ nosave=False, # do not save images/videos
+ classes=None, # filter by class: --class 0, or --class 0 2 3
+ agnostic_nms=False, # class-agnostic NMS
+ augment=False, # augmented inference
+ visualize=False, # visualize features
+ update=False, # update all models
+ project=ROOT / 'runs/detect', # save results to project/name
+ name='exp', # save results to project/name
+ exist_ok=False, # existing project/name ok, do not increment
+ line_thickness=3, # bounding box thickness (pixels)
+ hide_labels=False, # hide labels
+ hide_conf=False, # hide confidences
+ half=False, # use FP16 half-precision inference
+ dnn=False, # use OpenCV DNN for ONNX inference
+ vid_stride=1, # video frame-rate stride
+):
+ source = str(source)
+ save_img = not nosave and not source.endswith('.txt') # save inference images
+ is_file = Path(source).suffix[1:] in (IMG_FORMATS + VID_FORMATS)
+ is_url = source.lower().startswith(('rtsp://', 'rtmp://', 'http://', 'https://'))
+ webcam = source.isnumeric() or source.endswith('.txt') or (is_url and not is_file)
+ screenshot = source.lower().startswith('screen')
+ if is_url and is_file:
+ source = check_file(source) # download
+
+ # Directories
+ save_dir = increment_path(Path(project) / name, exist_ok=exist_ok) # increment run
+ (save_dir / 'labels' if save_txt else save_dir).mkdir(parents=True, exist_ok=True) # make dir
+
+ # Load model
+ device = select_device(device)
+ model = DetectMultiBackend(weights, device=device, dnn=dnn, data=data, fp16=half)
+ stride, names, pt = model.stride, model.names, model.pt
+ imgsz = check_img_size(imgsz, s=stride) # check image size
+
+ # Dataloader
+ bs = 1 # batch_size
+ if webcam:
+ view_img = check_imshow(warn=True)
+ dataset = LoadStreams(source, img_size=imgsz, stride=stride, auto=pt, vid_stride=vid_stride)
+ bs = len(dataset)
+ elif screenshot:
+ dataset = LoadScreenshots(source, img_size=imgsz, stride=stride, auto=pt)
+ else:
+ dataset = LoadImages(source, img_size=imgsz, stride=stride, auto=pt, vid_stride=vid_stride)
+ vid_path, vid_writer = [None] * bs, [None] * bs
+
+ # Run inference
+ model.warmup(imgsz=(1 if pt or model.triton else bs, 3, *imgsz)) # warmup
+ seen, windows, dt = 0, [], (Profile(), Profile(), Profile())
+ for path, im, im0s, vid_cap, s in dataset:
+ with dt[0]:
+ im = torch.from_numpy(im).to(model.device)
+ im = im.half() if model.fp16 else im.float() # uint8 to fp16/32
+ im /= 255 # 0 - 255 to 0.0 - 1.0
+ if len(im.shape) == 3:
+ im = im[None] # expand for batch dim
+
+ # Inference
+ with dt[1]:
+ visualize = increment_path(save_dir / Path(path).stem, mkdir=True) if visualize else False
+ pred = model(im, augment=augment, visualize=visualize)
+ pred = pred[0][1]
+
+ # NMS
+ with dt[2]:
+ pred = non_max_suppression(pred, conf_thres, iou_thres, classes, agnostic_nms, max_det=max_det)
+
+ # Second-stage classifier (optional)
+ # pred = utils.general.apply_classifier(pred, classifier_model, im, im0s)
+
+ # Process predictions
+ for i, det in enumerate(pred): # per image
+ seen += 1
+ if webcam: # batch_size >= 1
+ p, im0, frame = path[i], im0s[i].copy(), dataset.count
+ s += f'{i}: '
+ else:
+ p, im0, frame = path, im0s.copy(), getattr(dataset, 'frame', 0)
+
+ p = Path(p) # to Path
+ save_path = str(save_dir / p.name) # im.jpg
+ txt_path = str(save_dir / 'labels' / p.stem) + ('' if dataset.mode == 'image' else f'_{frame}') # im.txt
+ s += '%gx%g ' % im.shape[2:] # print string
+ gn = torch.tensor(im0.shape)[[1, 0, 1, 0]] # normalization gain whwh
+ imc = im0.copy() if save_crop else im0 # for save_crop
+ annotator = Annotator(im0, line_width=line_thickness, example=str(names))
+ if len(det):
+ # Rescale boxes from img_size to im0 size
+ det[:, :4] = scale_boxes(im.shape[2:], det[:, :4], im0.shape).round()
+
+ # Print results
+ for c in det[:, 5].unique():
+ n = (det[:, 5] == c).sum() # detections per class
+ s += f"{n} {names[int(c)]}{'s' * (n > 1)}, " # add to string
+
+ # Write results
+ for *xyxy, conf, cls in reversed(det):
+ if save_txt: # Write to file
+ xywh = (xyxy2xywh(torch.tensor(xyxy).view(1, 4)) / gn).view(-1).tolist() # normalized xywh
+ line = (cls, *xywh, conf) if save_conf else (cls, *xywh) # label format
+ with open(f'{txt_path}.txt', 'a') as f:
+ f.write(('%g ' * len(line)).rstrip() % line + '\n')
+
+ if save_img or save_crop or view_img: # Add bbox to image
+ c = int(cls) # integer class
+ label = None if hide_labels else (names[c] if hide_conf else f'{names[c]} {conf:.2f}')
+ annotator.box_label(xyxy, label, color=colors(c, True))
+ if save_crop:
+ save_one_box(xyxy, imc, file=save_dir / 'crops' / names[c] / f'{p.stem}.jpg', BGR=True)
+
+ # Stream results
+ im0 = annotator.result()
+ if view_img:
+ if platform.system() == 'Linux' and p not in windows:
+ windows.append(p)
+ cv2.namedWindow(str(p), cv2.WINDOW_NORMAL | cv2.WINDOW_KEEPRATIO) # allow window resize (Linux)
+ cv2.resizeWindow(str(p), im0.shape[1], im0.shape[0])
+ cv2.imshow(str(p), im0)
+ cv2.waitKey(1) # 1 millisecond
+
+ # Save results (image with detections)
+ if save_img:
+ if dataset.mode == 'image':
+ cv2.imwrite(save_path, im0)
+ else: # 'video' or 'stream'
+ if vid_path[i] != save_path: # new video
+ vid_path[i] = save_path
+ if isinstance(vid_writer[i], cv2.VideoWriter):
+ vid_writer[i].release() # release previous video writer
+ if vid_cap: # video
+ fps = vid_cap.get(cv2.CAP_PROP_FPS)
+ w = int(vid_cap.get(cv2.CAP_PROP_FRAME_WIDTH))
+ h = int(vid_cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
+ else: # stream
+ fps, w, h = 30, im0.shape[1], im0.shape[0]
+ save_path = str(Path(save_path).with_suffix('.mp4')) # force *.mp4 suffix on results videos
+ vid_writer[i] = cv2.VideoWriter(save_path, cv2.VideoWriter_fourcc(*'mp4v'), fps, (w, h))
+ vid_writer[i].write(im0)
+
+ # Print time (inference-only)
+ LOGGER.info(f"{s}{'' if len(det) else '(no detections), '}{dt[1].dt * 1E3:.1f}ms")
+
+ # Print results
+ t = tuple(x.t / seen * 1E3 for x in dt) # speeds per image
+ LOGGER.info(f'Speed: %.1fms pre-process, %.1fms inference, %.1fms NMS per image at shape {(1, 3, *imgsz)}' % t)
+ if save_txt or save_img:
+ s = f"\n{len(list(save_dir.glob('labels/*.txt')))} labels saved to {save_dir / 'labels'}" if save_txt else ''
+ LOGGER.info(f"Results saved to {colorstr('bold', save_dir)}{s}")
+ if update:
+ strip_optimizer(weights[0]) # update model (to fix SourceChangeWarning)
+
+
+def parse_opt():
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--weights', nargs='+', type=str, default=ROOT / 'yolo.pt', help='model path or triton URL')
+ parser.add_argument('--source', type=str, default=ROOT / 'data/images', help='file/dir/URL/glob/screen/0(webcam)')
+ parser.add_argument('--data', type=str, default=ROOT / 'data/coco128.yaml', help='(optional) dataset.yaml path')
+ parser.add_argument('--imgsz', '--img', '--img-size', nargs='+', type=int, default=[640], help='inference size h,w')
+ parser.add_argument('--conf-thres', type=float, default=0.25, help='confidence threshold')
+ parser.add_argument('--iou-thres', type=float, default=0.45, help='NMS IoU threshold')
+ parser.add_argument('--max-det', type=int, default=1000, help='maximum detections per image')
+ parser.add_argument('--device', default='', help='cuda device, i.e. 0 or 0,1,2,3 or cpu')
+ parser.add_argument('--view-img', action='store_true', help='show results')
+ parser.add_argument('--save-txt', action='store_true', help='save results to *.txt')
+ parser.add_argument('--save-conf', action='store_true', help='save confidences in --save-txt labels')
+ parser.add_argument('--save-crop', action='store_true', help='save cropped prediction boxes')
+ parser.add_argument('--nosave', action='store_true', help='do not save images/videos')
+ parser.add_argument('--classes', nargs='+', type=int, help='filter by class: --classes 0, or --classes 0 2 3')
+ parser.add_argument('--agnostic-nms', action='store_true', help='class-agnostic NMS')
+ parser.add_argument('--augment', action='store_true', help='augmented inference')
+ parser.add_argument('--visualize', action='store_true', help='visualize features')
+ parser.add_argument('--update', action='store_true', help='update all models')
+ parser.add_argument('--project', default=ROOT / 'runs/detect', help='save results to project/name')
+ parser.add_argument('--name', default='exp', help='save results to project/name')
+ parser.add_argument('--exist-ok', action='store_true', help='existing project/name ok, do not increment')
+ parser.add_argument('--line-thickness', default=3, type=int, help='bounding box thickness (pixels)')
+ parser.add_argument('--hide-labels', default=False, action='store_true', help='hide labels')
+ parser.add_argument('--hide-conf', default=False, action='store_true', help='hide confidences')
+ parser.add_argument('--half', action='store_true', help='use FP16 half-precision inference')
+ parser.add_argument('--dnn', action='store_true', help='use OpenCV DNN for ONNX inference')
+ parser.add_argument('--vid-stride', type=int, default=1, help='video frame-rate stride')
+ opt = parser.parse_args()
+ opt.imgsz *= 2 if len(opt.imgsz) == 1 else 1 # expand
+ print_args(vars(opt))
+ return opt
+
+
+def main(opt):
+ # check_requirements(exclude=('tensorboard', 'thop'))
+ run(**vars(opt))
+
+
+if __name__ == "__main__":
+ opt = parse_opt()
+ main(opt)
diff --git a/export.py b/export.py
new file mode 100644
index 0000000000000000000000000000000000000000..2ef415c1a79934d1d7101faff4106072a96d65cc
--- /dev/null
+++ b/export.py
@@ -0,0 +1,686 @@
+import argparse
+import contextlib
+import json
+import os
+import platform
+import re
+import subprocess
+import sys
+import time
+import warnings
+from pathlib import Path
+
+import pandas as pd
+import torch
+from torch.utils.mobile_optimizer import optimize_for_mobile
+
+FILE = Path(__file__).resolve()
+ROOT = FILE.parents[0] # YOLO root directory
+if str(ROOT) not in sys.path:
+ sys.path.append(str(ROOT)) # add ROOT to PATH
+if platform.system() != 'Windows':
+ ROOT = Path(os.path.relpath(ROOT, Path.cwd())) # relative
+
+from models.experimental import attempt_load, End2End
+from models.yolo import ClassificationModel, Detect, DDetect, DualDetect, DualDDetect, DetectionModel, SegmentationModel
+from utils.dataloaders import LoadImages
+from utils.general import (LOGGER, Profile, check_dataset, check_img_size, check_requirements, check_version,
+ check_yaml, colorstr, file_size, get_default_args, print_args, url2file, yaml_save)
+from utils.torch_utils import select_device, smart_inference_mode
+
+MACOS = platform.system() == 'Darwin' # macOS environment
+
+
+def export_formats():
+ # YOLO export formats
+ x = [
+ ['PyTorch', '-', '.pt', True, True],
+ ['TorchScript', 'torchscript', '.torchscript', True, True],
+ ['ONNX', 'onnx', '.onnx', True, True],
+ ['ONNX END2END', 'onnx_end2end', '_end2end.onnx', True, True],
+ ['OpenVINO', 'openvino', '_openvino_model', True, False],
+ ['TensorRT', 'engine', '.engine', False, True],
+ ['CoreML', 'coreml', '.mlmodel', True, False],
+ ['TensorFlow SavedModel', 'saved_model', '_saved_model', True, True],
+ ['TensorFlow GraphDef', 'pb', '.pb', True, True],
+ ['TensorFlow Lite', 'tflite', '.tflite', True, False],
+ ['TensorFlow Edge TPU', 'edgetpu', '_edgetpu.tflite', False, False],
+ ['TensorFlow.js', 'tfjs', '_web_model', False, False],
+ ['PaddlePaddle', 'paddle', '_paddle_model', True, True],]
+ return pd.DataFrame(x, columns=['Format', 'Argument', 'Suffix', 'CPU', 'GPU'])
+
+
+def try_export(inner_func):
+ # YOLO export decorator, i..e @try_export
+ inner_args = get_default_args(inner_func)
+
+ def outer_func(*args, **kwargs):
+ prefix = inner_args['prefix']
+ try:
+ with Profile() as dt:
+ f, model = inner_func(*args, **kwargs)
+ LOGGER.info(f'{prefix} export success ✅ {dt.t:.1f}s, saved as {f} ({file_size(f):.1f} MB)')
+ return f, model
+ except Exception as e:
+ LOGGER.info(f'{prefix} export failure ❌ {dt.t:.1f}s: {e}')
+ return None, None
+
+ return outer_func
+
+
+@try_export
+def export_torchscript(model, im, file, optimize, prefix=colorstr('TorchScript:')):
+ # YOLO TorchScript model export
+ LOGGER.info(f'\n{prefix} starting export with torch {torch.__version__}...')
+ f = file.with_suffix('.torchscript')
+
+ ts = torch.jit.trace(model, im, strict=False)
+ d = {"shape": im.shape, "stride": int(max(model.stride)), "names": model.names}
+ extra_files = {'config.txt': json.dumps(d)} # torch._C.ExtraFilesMap()
+ if optimize: # https://pytorch.org/tutorials/recipes/mobile_interpreter.html
+ optimize_for_mobile(ts)._save_for_lite_interpreter(str(f), _extra_files=extra_files)
+ else:
+ ts.save(str(f), _extra_files=extra_files)
+ return f, None
+
+
+@try_export
+def export_onnx(model, im, file, opset, dynamic, simplify, prefix=colorstr('ONNX:')):
+ # YOLO ONNX export
+ check_requirements('onnx')
+ import onnx
+
+ LOGGER.info(f'\n{prefix} starting export with onnx {onnx.__version__}...')
+ f = file.with_suffix('.onnx')
+
+ output_names = ['output0', 'output1'] if isinstance(model, SegmentationModel) else ['output0']
+ if dynamic:
+ dynamic = {'images': {0: 'batch', 2: 'height', 3: 'width'}} # shape(1,3,640,640)
+ if isinstance(model, SegmentationModel):
+ dynamic['output0'] = {0: 'batch', 1: 'anchors'} # shape(1,25200,85)
+ dynamic['output1'] = {0: 'batch', 2: 'mask_height', 3: 'mask_width'} # shape(1,32,160,160)
+ elif isinstance(model, DetectionModel):
+ dynamic['output0'] = {0: 'batch', 1: 'anchors'} # shape(1,25200,85)
+
+ torch.onnx.export(
+ model.cpu() if dynamic else model, # --dynamic only compatible with cpu
+ im.cpu() if dynamic else im,
+ f,
+ verbose=False,
+ opset_version=opset,
+ do_constant_folding=True,
+ input_names=['images'],
+ output_names=output_names,
+ dynamic_axes=dynamic or None)
+
+ # Checks
+ model_onnx = onnx.load(f) # load onnx model
+ onnx.checker.check_model(model_onnx) # check onnx model
+
+ # Metadata
+ d = {'stride': int(max(model.stride)), 'names': model.names}
+ for k, v in d.items():
+ meta = model_onnx.metadata_props.add()
+ meta.key, meta.value = k, str(v)
+ onnx.save(model_onnx, f)
+
+ # Simplify
+ if simplify:
+ try:
+ cuda = torch.cuda.is_available()
+ check_requirements(('onnxruntime-gpu' if cuda else 'onnxruntime', 'onnx-simplifier>=0.4.1'))
+ import onnxsim
+
+ LOGGER.info(f'{prefix} simplifying with onnx-simplifier {onnxsim.__version__}...')
+ model_onnx, check = onnxsim.simplify(model_onnx)
+ assert check, 'assert check failed'
+ onnx.save(model_onnx, f)
+ except Exception as e:
+ LOGGER.info(f'{prefix} simplifier failure: {e}')
+ return f, model_onnx
+
+
+@try_export
+def export_onnx_end2end(model, im, file, simplify, topk_all, iou_thres, conf_thres, device, labels, prefix=colorstr('ONNX END2END:')):
+ # YOLO ONNX export
+ check_requirements('onnx')
+ import onnx
+ LOGGER.info(f'\n{prefix} starting export with onnx {onnx.__version__}...')
+ f = os.path.splitext(file)[0] + "-end2end.onnx"
+ batch_size = 'batch'
+
+ dynamic_axes = {'images': {0 : 'batch', 2: 'height', 3:'width'}, } # variable length axes
+
+ output_axes = {
+ 'num_dets': {0: 'batch'},
+ 'det_boxes': {0: 'batch'},
+ 'det_scores': {0: 'batch'},
+ 'det_classes': {0: 'batch'},
+ }
+ dynamic_axes.update(output_axes)
+ model = End2End(model, topk_all, iou_thres, conf_thres, None ,device, labels)
+
+ output_names = ['num_dets', 'det_boxes', 'det_scores', 'det_classes']
+ shapes = [ batch_size, 1, batch_size, topk_all, 4,
+ batch_size, topk_all, batch_size, topk_all]
+
+ torch.onnx.export(model,
+ im,
+ f,
+ verbose=False,
+ export_params=True, # store the trained parameter weights inside the model file
+ opset_version=12,
+ do_constant_folding=True, # whether to execute constant folding for optimization
+ input_names=['images'],
+ output_names=output_names,
+ dynamic_axes=dynamic_axes)
+
+ # Checks
+ model_onnx = onnx.load(f) # load onnx model
+ onnx.checker.check_model(model_onnx) # check onnx model
+ for i in model_onnx.graph.output:
+ for j in i.type.tensor_type.shape.dim:
+ j.dim_param = str(shapes.pop(0))
+
+ if simplify:
+ try:
+ import onnxsim
+
+ print('\nStarting to simplify ONNX...')
+ model_onnx, check = onnxsim.simplify(model_onnx)
+ assert check, 'assert check failed'
+ except Exception as e:
+ print(f'Simplifier failure: {e}')
+
+ # print(onnx.helper.printable_graph(onnx_model.graph)) # print a human readable model
+ onnx.save(model_onnx,f)
+ print('ONNX export success, saved as %s' % f)
+ return f, model_onnx
+
+
+@try_export
+def export_openvino(file, metadata, half, prefix=colorstr('OpenVINO:')):
+ # YOLO OpenVINO export
+ check_requirements('openvino-dev') # requires openvino-dev: https://pypi.org/project/openvino-dev/
+ import openvino.inference_engine as ie
+
+ LOGGER.info(f'\n{prefix} starting export with openvino {ie.__version__}...')
+ f = str(file).replace('.pt', f'_openvino_model{os.sep}')
+
+ #cmd = f"mo --input_model {file.with_suffix('.onnx')} --output_dir {f} --data_type {'FP16' if half else 'FP32'}"
+ #cmd = f"mo --input_model {file.with_suffix('.onnx')} --output_dir {f} {"--compress_to_fp16" if half else ""}"
+ half_arg = "--compress_to_fp16" if half else ""
+ cmd = f"mo --input_model {file.with_suffix('.onnx')} --output_dir {f} {half_arg}"
+ subprocess.run(cmd.split(), check=True, env=os.environ) # export
+ yaml_save(Path(f) / file.with_suffix('.yaml').name, metadata) # add metadata.yaml
+ return f, None
+
+
+@try_export
+def export_paddle(model, im, file, metadata, prefix=colorstr('PaddlePaddle:')):
+ # YOLO Paddle export
+ check_requirements(('paddlepaddle', 'x2paddle'))
+ import x2paddle
+ from x2paddle.convert import pytorch2paddle
+
+ LOGGER.info(f'\n{prefix} starting export with X2Paddle {x2paddle.__version__}...')
+ f = str(file).replace('.pt', f'_paddle_model{os.sep}')
+
+ pytorch2paddle(module=model, save_dir=f, jit_type='trace', input_examples=[im]) # export
+ yaml_save(Path(f) / file.with_suffix('.yaml').name, metadata) # add metadata.yaml
+ return f, None
+
+
+@try_export
+def export_coreml(model, im, file, int8, half, prefix=colorstr('CoreML:')):
+ # YOLO CoreML export
+ check_requirements('coremltools')
+ import coremltools as ct
+
+ LOGGER.info(f'\n{prefix} starting export with coremltools {ct.__version__}...')
+ f = file.with_suffix('.mlmodel')
+
+ ts = torch.jit.trace(model, im, strict=False) # TorchScript model
+ ct_model = ct.convert(ts, inputs=[ct.ImageType('image', shape=im.shape, scale=1 / 255, bias=[0, 0, 0])])
+ bits, mode = (8, 'kmeans_lut') if int8 else (16, 'linear') if half else (32, None)
+ if bits < 32:
+ if MACOS: # quantization only supported on macOS
+ with warnings.catch_warnings():
+ warnings.filterwarnings("ignore", category=DeprecationWarning) # suppress numpy==1.20 float warning
+ ct_model = ct.models.neural_network.quantization_utils.quantize_weights(ct_model, bits, mode)
+ else:
+ print(f'{prefix} quantization only supported on macOS, skipping...')
+ ct_model.save(f)
+ return f, ct_model
+
+
+@try_export
+def export_engine(model, im, file, half, dynamic, simplify, workspace=4, verbose=False, prefix=colorstr('TensorRT:')):
+ # YOLO TensorRT export https://developer.nvidia.com/tensorrt
+ assert im.device.type != 'cpu', 'export running on CPU but must be on GPU, i.e. `python export.py --device 0`'
+ try:
+ import tensorrt as trt
+ except Exception:
+ if platform.system() == 'Linux':
+ check_requirements('nvidia-tensorrt', cmds='-U --index-url https://pypi.ngc.nvidia.com')
+ import tensorrt as trt
+
+ if trt.__version__[0] == '7': # TensorRT 7 handling https://github.com/ultralytics/yolov5/issues/6012
+ grid = model.model[-1].anchor_grid
+ model.model[-1].anchor_grid = [a[..., :1, :1, :] for a in grid]
+ export_onnx(model, im, file, 12, dynamic, simplify) # opset 12
+ model.model[-1].anchor_grid = grid
+ else: # TensorRT >= 8
+ check_version(trt.__version__, '8.0.0', hard=True) # require tensorrt>=8.0.0
+ export_onnx(model, im, file, 12, dynamic, simplify) # opset 12
+ onnx = file.with_suffix('.onnx')
+
+ LOGGER.info(f'\n{prefix} starting export with TensorRT {trt.__version__}...')
+ assert onnx.exists(), f'failed to export ONNX file: {onnx}'
+ f = file.with_suffix('.engine') # TensorRT engine file
+ logger = trt.Logger(trt.Logger.INFO)
+ if verbose:
+ logger.min_severity = trt.Logger.Severity.VERBOSE
+
+ builder = trt.Builder(logger)
+ config = builder.create_builder_config()
+ config.max_workspace_size = workspace * 1 << 30
+ # config.set_memory_pool_limit(trt.MemoryPoolType.WORKSPACE, workspace << 30) # fix TRT 8.4 deprecation notice
+
+ flag = (1 << int(trt.NetworkDefinitionCreationFlag.EXPLICIT_BATCH))
+ network = builder.create_network(flag)
+ parser = trt.OnnxParser(network, logger)
+ if not parser.parse_from_file(str(onnx)):
+ raise RuntimeError(f'failed to load ONNX file: {onnx}')
+
+ inputs = [network.get_input(i) for i in range(network.num_inputs)]
+ outputs = [network.get_output(i) for i in range(network.num_outputs)]
+ for inp in inputs:
+ LOGGER.info(f'{prefix} input "{inp.name}" with shape{inp.shape} {inp.dtype}')
+ for out in outputs:
+ LOGGER.info(f'{prefix} output "{out.name}" with shape{out.shape} {out.dtype}')
+
+ if dynamic:
+ if im.shape[0] <= 1:
+ LOGGER.warning(f"{prefix} WARNING ⚠️ --dynamic model requires maximum --batch-size argument")
+ profile = builder.create_optimization_profile()
+ for inp in inputs:
+ profile.set_shape(inp.name, (1, *im.shape[1:]), (max(1, im.shape[0] // 2), *im.shape[1:]), im.shape)
+ config.add_optimization_profile(profile)
+
+ LOGGER.info(f'{prefix} building FP{16 if builder.platform_has_fast_fp16 and half else 32} engine as {f}')
+ if builder.platform_has_fast_fp16 and half:
+ config.set_flag(trt.BuilderFlag.FP16)
+ with builder.build_engine(network, config) as engine, open(f, 'wb') as t:
+ t.write(engine.serialize())
+ return f, None
+
+
+@try_export
+def export_saved_model(model,
+ im,
+ file,
+ dynamic,
+ tf_nms=False,
+ agnostic_nms=False,
+ topk_per_class=100,
+ topk_all=100,
+ iou_thres=0.45,
+ conf_thres=0.25,
+ keras=False,
+ prefix=colorstr('TensorFlow SavedModel:')):
+ # YOLO TensorFlow SavedModel export
+ try:
+ import tensorflow as tf
+ except Exception:
+ check_requirements(f"tensorflow{'' if torch.cuda.is_available() else '-macos' if MACOS else '-cpu'}")
+ import tensorflow as tf
+ from tensorflow.python.framework.convert_to_constants import convert_variables_to_constants_v2
+
+ from models.tf import TFModel
+
+ LOGGER.info(f'\n{prefix} starting export with tensorflow {tf.__version__}...')
+ f = str(file).replace('.pt', '_saved_model')
+ batch_size, ch, *imgsz = list(im.shape) # BCHW
+
+ tf_model = TFModel(cfg=model.yaml, model=model, nc=model.nc, imgsz=imgsz)
+ im = tf.zeros((batch_size, *imgsz, ch)) # BHWC order for TensorFlow
+ _ = tf_model.predict(im, tf_nms, agnostic_nms, topk_per_class, topk_all, iou_thres, conf_thres)
+ inputs = tf.keras.Input(shape=(*imgsz, ch), batch_size=None if dynamic else batch_size)
+ outputs = tf_model.predict(inputs, tf_nms, agnostic_nms, topk_per_class, topk_all, iou_thres, conf_thres)
+ keras_model = tf.keras.Model(inputs=inputs, outputs=outputs)
+ keras_model.trainable = False
+ keras_model.summary()
+ if keras:
+ keras_model.save(f, save_format='tf')
+ else:
+ spec = tf.TensorSpec(keras_model.inputs[0].shape, keras_model.inputs[0].dtype)
+ m = tf.function(lambda x: keras_model(x)) # full model
+ m = m.get_concrete_function(spec)
+ frozen_func = convert_variables_to_constants_v2(m)
+ tfm = tf.Module()
+ tfm.__call__ = tf.function(lambda x: frozen_func(x)[:4] if tf_nms else frozen_func(x), [spec])
+ tfm.__call__(im)
+ tf.saved_model.save(tfm,
+ f,
+ options=tf.saved_model.SaveOptions(experimental_custom_gradients=False) if check_version(
+ tf.__version__, '2.6') else tf.saved_model.SaveOptions())
+ return f, keras_model
+
+
+@try_export
+def export_pb(keras_model, file, prefix=colorstr('TensorFlow GraphDef:')):
+ # YOLO TensorFlow GraphDef *.pb export https://github.com/leimao/Frozen_Graph_TensorFlow
+ import tensorflow as tf
+ from tensorflow.python.framework.convert_to_constants import convert_variables_to_constants_v2
+
+ LOGGER.info(f'\n{prefix} starting export with tensorflow {tf.__version__}...')
+ f = file.with_suffix('.pb')
+
+ m = tf.function(lambda x: keras_model(x)) # full model
+ m = m.get_concrete_function(tf.TensorSpec(keras_model.inputs[0].shape, keras_model.inputs[0].dtype))
+ frozen_func = convert_variables_to_constants_v2(m)
+ frozen_func.graph.as_graph_def()
+ tf.io.write_graph(graph_or_graph_def=frozen_func.graph, logdir=str(f.parent), name=f.name, as_text=False)
+ return f, None
+
+
+@try_export
+def export_tflite(keras_model, im, file, int8, data, nms, agnostic_nms, prefix=colorstr('TensorFlow Lite:')):
+ # YOLOv5 TensorFlow Lite export
+ import tensorflow as tf
+
+ LOGGER.info(f'\n{prefix} starting export with tensorflow {tf.__version__}...')
+ batch_size, ch, *imgsz = list(im.shape) # BCHW
+ f = str(file).replace('.pt', '-fp16.tflite')
+
+ converter = tf.lite.TFLiteConverter.from_keras_model(keras_model)
+ converter.target_spec.supported_ops = [tf.lite.OpsSet.TFLITE_BUILTINS]
+ converter.target_spec.supported_types = [tf.float16]
+ converter.optimizations = [tf.lite.Optimize.DEFAULT]
+ if int8:
+ from models.tf import representative_dataset_gen
+ dataset = LoadImages(check_dataset(check_yaml(data))['train'], img_size=imgsz, auto=False)
+ converter.representative_dataset = lambda: representative_dataset_gen(dataset, ncalib=100)
+ converter.target_spec.supported_ops = [tf.lite.OpsSet.TFLITE_BUILTINS_INT8]
+ converter.target_spec.supported_types = []
+ converter.inference_input_type = tf.uint8 # or tf.int8
+ converter.inference_output_type = tf.uint8 # or tf.int8
+ converter.experimental_new_quantizer = True
+ f = str(file).replace('.pt', '-int8.tflite')
+ if nms or agnostic_nms:
+ converter.target_spec.supported_ops.append(tf.lite.OpsSet.SELECT_TF_OPS)
+
+ tflite_model = converter.convert()
+ open(f, "wb").write(tflite_model)
+ return f, None
+
+
+@try_export
+def export_edgetpu(file, prefix=colorstr('Edge TPU:')):
+ # YOLO Edge TPU export https://coral.ai/docs/edgetpu/models-intro/
+ cmd = 'edgetpu_compiler --version'
+ help_url = 'https://coral.ai/docs/edgetpu/compiler/'
+ assert platform.system() == 'Linux', f'export only supported on Linux. See {help_url}'
+ if subprocess.run(f'{cmd} >/dev/null', shell=True).returncode != 0:
+ LOGGER.info(f'\n{prefix} export requires Edge TPU compiler. Attempting install from {help_url}')
+ sudo = subprocess.run('sudo --version >/dev/null', shell=True).returncode == 0 # sudo installed on system
+ for c in (
+ 'curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -',
+ 'echo "deb https://packages.cloud.google.com/apt coral-edgetpu-stable main" | sudo tee /etc/apt/sources.list.d/coral-edgetpu.list',
+ 'sudo apt-get update', 'sudo apt-get install edgetpu-compiler'):
+ subprocess.run(c if sudo else c.replace('sudo ', ''), shell=True, check=True)
+ ver = subprocess.run(cmd, shell=True, capture_output=True, check=True).stdout.decode().split()[-1]
+
+ LOGGER.info(f'\n{prefix} starting export with Edge TPU compiler {ver}...')
+ f = str(file).replace('.pt', '-int8_edgetpu.tflite') # Edge TPU model
+ f_tfl = str(file).replace('.pt', '-int8.tflite') # TFLite model
+
+ cmd = f"edgetpu_compiler -s -d -k 10 --out_dir {file.parent} {f_tfl}"
+ subprocess.run(cmd.split(), check=True)
+ return f, None
+
+
+@try_export
+def export_tfjs(file, prefix=colorstr('TensorFlow.js:')):
+ # YOLO TensorFlow.js export
+ check_requirements('tensorflowjs')
+ import tensorflowjs as tfjs
+
+ LOGGER.info(f'\n{prefix} starting export with tensorflowjs {tfjs.__version__}...')
+ f = str(file).replace('.pt', '_web_model') # js dir
+ f_pb = file.with_suffix('.pb') # *.pb path
+ f_json = f'{f}/model.json' # *.json path
+
+ cmd = f'tensorflowjs_converter --input_format=tf_frozen_model ' \
+ f'--output_node_names=Identity,Identity_1,Identity_2,Identity_3 {f_pb} {f}'
+ subprocess.run(cmd.split())
+
+ json = Path(f_json).read_text()
+ with open(f_json, 'w') as j: # sort JSON Identity_* in ascending order
+ subst = re.sub(
+ r'{"outputs": {"Identity.?.?": {"name": "Identity.?.?"}, '
+ r'"Identity.?.?": {"name": "Identity.?.?"}, '
+ r'"Identity.?.?": {"name": "Identity.?.?"}, '
+ r'"Identity.?.?": {"name": "Identity.?.?"}}}', r'{"outputs": {"Identity": {"name": "Identity"}, '
+ r'"Identity_1": {"name": "Identity_1"}, '
+ r'"Identity_2": {"name": "Identity_2"}, '
+ r'"Identity_3": {"name": "Identity_3"}}}', json)
+ j.write(subst)
+ return f, None
+
+
+def add_tflite_metadata(file, metadata, num_outputs):
+ # Add metadata to *.tflite models per https://www.tensorflow.org/lite/models/convert/metadata
+ with contextlib.suppress(ImportError):
+ # check_requirements('tflite_support')
+ from tflite_support import flatbuffers
+ from tflite_support import metadata as _metadata
+ from tflite_support import metadata_schema_py_generated as _metadata_fb
+
+ tmp_file = Path('/tmp/meta.txt')
+ with open(tmp_file, 'w') as meta_f:
+ meta_f.write(str(metadata))
+
+ model_meta = _metadata_fb.ModelMetadataT()
+ label_file = _metadata_fb.AssociatedFileT()
+ label_file.name = tmp_file.name
+ model_meta.associatedFiles = [label_file]
+
+ subgraph = _metadata_fb.SubGraphMetadataT()
+ subgraph.inputTensorMetadata = [_metadata_fb.TensorMetadataT()]
+ subgraph.outputTensorMetadata = [_metadata_fb.TensorMetadataT()] * num_outputs
+ model_meta.subgraphMetadata = [subgraph]
+
+ b = flatbuffers.Builder(0)
+ b.Finish(model_meta.Pack(b), _metadata.MetadataPopulator.METADATA_FILE_IDENTIFIER)
+ metadata_buf = b.Output()
+
+ populator = _metadata.MetadataPopulator.with_model_file(file)
+ populator.load_metadata_buffer(metadata_buf)
+ populator.load_associated_files([str(tmp_file)])
+ populator.populate()
+ tmp_file.unlink()
+
+
+@smart_inference_mode()
+def run(
+ data=ROOT / 'data/coco.yaml', # 'dataset.yaml path'
+ weights=ROOT / 'yolo.pt', # weights path
+ imgsz=(640, 640), # image (height, width)
+ batch_size=1, # batch size
+ device='cpu', # cuda device, i.e. 0 or 0,1,2,3 or cpu
+ include=('torchscript', 'onnx'), # include formats
+ half=False, # FP16 half-precision export
+ inplace=False, # set YOLO Detect() inplace=True
+ keras=False, # use Keras
+ optimize=False, # TorchScript: optimize for mobile
+ int8=False, # CoreML/TF INT8 quantization
+ dynamic=False, # ONNX/TF/TensorRT: dynamic axes
+ simplify=False, # ONNX: simplify model
+ opset=12, # ONNX: opset version
+ verbose=False, # TensorRT: verbose log
+ workspace=4, # TensorRT: workspace size (GB)
+ nms=False, # TF: add NMS to model
+ agnostic_nms=False, # TF: add agnostic NMS to model
+ topk_per_class=100, # TF.js NMS: topk per class to keep
+ topk_all=100, # TF.js NMS: topk for all classes to keep
+ iou_thres=0.45, # TF.js NMS: IoU threshold
+ conf_thres=0.25, # TF.js NMS: confidence threshold
+):
+ t = time.time()
+ include = [x.lower() for x in include] # to lowercase
+ fmts = tuple(export_formats()['Argument'][1:]) # --include arguments
+ flags = [x in include for x in fmts]
+ assert sum(flags) == len(include), f'ERROR: Invalid --include {include}, valid --include arguments are {fmts}'
+ jit, onnx, onnx_end2end, xml, engine, coreml, saved_model, pb, tflite, edgetpu, tfjs, paddle = flags # export booleans
+ file = Path(url2file(weights) if str(weights).startswith(('http:/', 'https:/')) else weights) # PyTorch weights
+
+ # Load PyTorch model
+ device = select_device(device)
+ if half:
+ assert device.type != 'cpu' or coreml, '--half only compatible with GPU export, i.e. use --device 0'
+ assert not dynamic, '--half not compatible with --dynamic, i.e. use either --half or --dynamic but not both'
+ model = attempt_load(weights, device=device, inplace=True, fuse=True) # load FP32 model
+
+ # Checks
+ imgsz *= 2 if len(imgsz) == 1 else 1 # expand
+ if optimize:
+ assert device.type == 'cpu', '--optimize not compatible with cuda devices, i.e. use --device cpu'
+
+ # Input
+ gs = int(max(model.stride)) # grid size (max stride)
+ imgsz = [check_img_size(x, gs) for x in imgsz] # verify img_size are gs-multiples
+ im = torch.zeros(batch_size, 3, *imgsz).to(device) # image size(1,3,320,192) BCHW iDetection
+
+ # Update model
+ model.eval()
+ for k, m in model.named_modules():
+ if isinstance(m, (Detect, DDetect, DualDetect, DualDDetect)):
+ m.inplace = inplace
+ m.dynamic = dynamic
+ m.export = True
+
+ for _ in range(2):
+ y = model(im) # dry runs
+ if half and not coreml:
+ im, model = im.half(), model.half() # to FP16
+ shape = tuple((y[0] if isinstance(y, (tuple, list)) else y).shape) # model output shape
+ metadata = {'stride': int(max(model.stride)), 'names': model.names} # model metadata
+ LOGGER.info(f"\n{colorstr('PyTorch:')} starting from {file} with output shape {shape} ({file_size(file):.1f} MB)")
+
+ # Exports
+ f = [''] * len(fmts) # exported filenames
+ warnings.filterwarnings(action='ignore', category=torch.jit.TracerWarning) # suppress TracerWarning
+ if jit: # TorchScript
+ f[0], _ = export_torchscript(model, im, file, optimize)
+ if engine: # TensorRT required before ONNX
+ f[1], _ = export_engine(model, im, file, half, dynamic, simplify, workspace, verbose)
+ if onnx or xml: # OpenVINO requires ONNX
+ f[2], _ = export_onnx(model, im, file, opset, dynamic, simplify)
+ if onnx_end2end:
+ if isinstance(model, DetectionModel):
+ labels = model.names
+ f[2], _ = export_onnx_end2end(model, im, file, simplify, topk_all, iou_thres, conf_thres, device, len(labels))
+ else:
+ raise RuntimeError("The model is not a DetectionModel.")
+ if xml: # OpenVINO
+ f[3], _ = export_openvino(file, metadata, half)
+ if coreml: # CoreML
+ f[4], _ = export_coreml(model, im, file, int8, half)
+ if any((saved_model, pb, tflite, edgetpu, tfjs)): # TensorFlow formats
+ assert not tflite or not tfjs, 'TFLite and TF.js models must be exported separately, please pass only one type.'
+ assert not isinstance(model, ClassificationModel), 'ClassificationModel export to TF formats not yet supported.'
+ f[5], s_model = export_saved_model(model.cpu(),
+ im,
+ file,
+ dynamic,
+ tf_nms=nms or agnostic_nms or tfjs,
+ agnostic_nms=agnostic_nms or tfjs,
+ topk_per_class=topk_per_class,
+ topk_all=topk_all,
+ iou_thres=iou_thres,
+ conf_thres=conf_thres,
+ keras=keras)
+ if pb or tfjs: # pb prerequisite to tfjs
+ f[6], _ = export_pb(s_model, file)
+ if tflite or edgetpu:
+ f[7], _ = export_tflite(s_model, im, file, int8 or edgetpu, data=data, nms=nms, agnostic_nms=agnostic_nms)
+ if edgetpu:
+ f[8], _ = export_edgetpu(file)
+ add_tflite_metadata(f[8] or f[7], metadata, num_outputs=len(s_model.outputs))
+ if tfjs:
+ f[9], _ = export_tfjs(file)
+ if paddle: # PaddlePaddle
+ f[10], _ = export_paddle(model, im, file, metadata)
+
+ # Finish
+ f = [str(x) for x in f if x] # filter out '' and None
+ if any(f):
+ cls, det, seg = (isinstance(model, x) for x in (ClassificationModel, DetectionModel, SegmentationModel)) # type
+ dir = Path('segment' if seg else 'classify' if cls else '')
+ h = '--half' if half else '' # --half FP16 inference arg
+ s = "# WARNING ⚠️ ClassificationModel not yet supported for PyTorch Hub AutoShape inference" if cls else \
+ "# WARNING ⚠️ SegmentationModel not yet supported for PyTorch Hub AutoShape inference" if seg else ''
+ if onnx_end2end:
+ LOGGER.info(f'\nExport complete ({time.time() - t:.1f}s)'
+ f"\nResults saved to {colorstr('bold', file.parent.resolve())}"
+ f"\nVisualize: https://netron.app")
+ else:
+ LOGGER.info(f'\nExport complete ({time.time() - t:.1f}s)'
+ f"\nResults saved to {colorstr('bold', file.parent.resolve())}"
+ f"\nDetect: python {dir / ('detect.py' if det else 'predict.py')} --weights {f[-1]} {h}"
+ f"\nValidate: python {dir / 'val.py'} --weights {f[-1]} {h}"
+ f"\nPyTorch Hub: model = torch.hub.load('ultralytics/yolov5', 'custom', '{f[-1]}') {s}"
+ f"\nVisualize: https://netron.app")
+ return f # return list of exported files/dirs
+
+
+def parse_opt():
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--data', type=str, default=ROOT / 'data/coco.yaml', help='dataset.yaml path')
+ parser.add_argument('--weights', nargs='+', type=str, default=ROOT / 'yolo.pt', help='model.pt path(s)')
+ parser.add_argument('--imgsz', '--img', '--img-size', nargs='+', type=int, default=[640, 640], help='image (h, w)')
+ parser.add_argument('--batch-size', type=int, default=1, help='batch size')
+ parser.add_argument('--device', default='cpu', help='cuda device, i.e. 0 or 0,1,2,3 or cpu')
+ parser.add_argument('--half', action='store_true', help='FP16 half-precision export')
+ parser.add_argument('--inplace', action='store_true', help='set YOLO Detect() inplace=True')
+ parser.add_argument('--keras', action='store_true', help='TF: use Keras')
+ parser.add_argument('--optimize', action='store_true', help='TorchScript: optimize for mobile')
+ parser.add_argument('--int8', action='store_true', help='CoreML/TF INT8 quantization')
+ parser.add_argument('--dynamic', action='store_true', help='ONNX/TF/TensorRT: dynamic axes')
+ parser.add_argument('--simplify', action='store_true', help='ONNX: simplify model')
+ parser.add_argument('--opset', type=int, default=12, help='ONNX: opset version')
+ parser.add_argument('--verbose', action='store_true', help='TensorRT: verbose log')
+ parser.add_argument('--workspace', type=int, default=4, help='TensorRT: workspace size (GB)')
+ parser.add_argument('--nms', action='store_true', help='TF: add NMS to model')
+ parser.add_argument('--agnostic-nms', action='store_true', help='TF: add agnostic NMS to model')
+ parser.add_argument('--topk-per-class', type=int, default=100, help='TF.js NMS: topk per class to keep')
+ parser.add_argument('--topk-all', type=int, default=100, help='ONNX END2END/TF.js NMS: topk for all classes to keep')
+ parser.add_argument('--iou-thres', type=float, default=0.45, help='ONNX END2END/TF.js NMS: IoU threshold')
+ parser.add_argument('--conf-thres', type=float, default=0.25, help='ONNX END2END/TF.js NMS: confidence threshold')
+ parser.add_argument(
+ '--include',
+ nargs='+',
+ default=['torchscript'],
+ help='torchscript, onnx, onnx_end2end, openvino, engine, coreml, saved_model, pb, tflite, edgetpu, tfjs, paddle')
+ opt = parser.parse_args()
+
+ if 'onnx_end2end' in opt.include:
+ opt.simplify = True
+ opt.dynamic = True
+ opt.inplace = True
+ opt.half = False
+
+ print_args(vars(opt))
+ return opt
+
+
+def main(opt):
+ for opt.weights in (opt.weights if isinstance(opt.weights, list) else [opt.weights]):
+ run(**vars(opt))
+
+
+if __name__ == "__main__":
+ opt = parse_opt()
+ main(opt)
diff --git a/figure/horses_prediction.jpg b/figure/horses_prediction.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..0fbfc83f8ef44a6e6ef170d70a73980de078e5db
Binary files /dev/null and b/figure/horses_prediction.jpg differ
diff --git a/figure/multitask.png b/figure/multitask.png
new file mode 100644
index 0000000000000000000000000000000000000000..7dad29ffbf5279feb4ef023141de282f4211877a
--- /dev/null
+++ b/figure/multitask.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b7c83ee5db84a3760a0f854e5d70ed0e2ca1cc0f5ef5ff8a88e87d525e87eee1
+size 1292320
diff --git a/figure/performance.png b/figure/performance.png
new file mode 100644
index 0000000000000000000000000000000000000000..572f3e02d474a72e1344d38e186da558cb3eb212
Binary files /dev/null and b/figure/performance.png differ
diff --git a/hubconf.py b/hubconf.py
new file mode 100644
index 0000000000000000000000000000000000000000..b4d6b6e4180b6e71bf24908528558e5e266b378e
--- /dev/null
+++ b/hubconf.py
@@ -0,0 +1,107 @@
+import torch
+
+
+def _create(name, pretrained=True, channels=3, classes=80, autoshape=True, verbose=True, device=None):
+ """Creates or loads a YOLO model
+
+ Arguments:
+ name (str): model name 'yolov3' or path 'path/to/best.pt'
+ pretrained (bool): load pretrained weights into the model
+ channels (int): number of input channels
+ classes (int): number of model classes
+ autoshape (bool): apply YOLO .autoshape() wrapper to model
+ verbose (bool): print all information to screen
+ device (str, torch.device, None): device to use for model parameters
+
+ Returns:
+ YOLO model
+ """
+ from pathlib import Path
+
+ from models.common import AutoShape, DetectMultiBackend
+ from models.experimental import attempt_load
+ from models.yolo import ClassificationModel, DetectionModel, SegmentationModel
+ from utils.downloads import attempt_download
+ from utils.general import LOGGER, check_requirements, intersect_dicts, logging
+ from utils.torch_utils import select_device
+
+ if not verbose:
+ LOGGER.setLevel(logging.WARNING)
+ check_requirements(exclude=('opencv-python', 'tensorboard', 'thop'))
+ name = Path(name)
+ path = name.with_suffix('.pt') if name.suffix == '' and not name.is_dir() else name # checkpoint path
+ try:
+ device = select_device(device)
+ if pretrained and channels == 3 and classes == 80:
+ try:
+ model = DetectMultiBackend(path, device=device, fuse=autoshape) # detection model
+ if autoshape:
+ if model.pt and isinstance(model.model, ClassificationModel):
+ LOGGER.warning('WARNING ⚠️ YOLO ClassificationModel is not yet AutoShape compatible. '
+ 'You must pass torch tensors in BCHW to this model, i.e. shape(1,3,224,224).')
+ elif model.pt and isinstance(model.model, SegmentationModel):
+ LOGGER.warning('WARNING ⚠️ YOLO SegmentationModel is not yet AutoShape compatible. '
+ 'You will not be able to run inference with this model.')
+ else:
+ model = AutoShape(model) # for file/URI/PIL/cv2/np inputs and NMS
+ except Exception:
+ model = attempt_load(path, device=device, fuse=False) # arbitrary model
+ else:
+ cfg = list((Path(__file__).parent / 'models').rglob(f'{path.stem}.yaml'))[0] # model.yaml path
+ model = DetectionModel(cfg, channels, classes) # create model
+ if pretrained:
+ ckpt = torch.load(attempt_download(path), map_location=device) # load
+ csd = ckpt['model'].float().state_dict() # checkpoint state_dict as FP32
+ csd = intersect_dicts(csd, model.state_dict(), exclude=['anchors']) # intersect
+ model.load_state_dict(csd, strict=False) # load
+ if len(ckpt['model'].names) == classes:
+ model.names = ckpt['model'].names # set class names attribute
+ if not verbose:
+ LOGGER.setLevel(logging.INFO) # reset to default
+ return model.to(device)
+
+ except Exception as e:
+ help_url = 'https://github.com/ultralytics/yolov5/issues/36'
+ s = f'{e}. Cache may be out of date, try `force_reload=True` or see {help_url} for help.'
+ raise Exception(s) from e
+
+
+def custom(path='path/to/model.pt', autoshape=True, _verbose=True, device=None):
+ # YOLO custom or local model
+ return _create(path, autoshape=autoshape, verbose=_verbose, device=device)
+
+
+if __name__ == '__main__':
+ import argparse
+ from pathlib import Path
+
+ import numpy as np
+ from PIL import Image
+
+ from utils.general import cv2, print_args
+
+ # Argparser
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--model', type=str, default='yolo', help='model name')
+ opt = parser.parse_args()
+ print_args(vars(opt))
+
+ # Model
+ model = _create(name=opt.model, pretrained=True, channels=3, classes=80, autoshape=True, verbose=True)
+ # model = custom(path='path/to/model.pt') # custom
+
+ # Images
+ imgs = [
+ 'data/images/zidane.jpg', # filename
+ Path('data/images/zidane.jpg'), # Path
+ 'https://ultralytics.com/images/zidane.jpg', # URI
+ cv2.imread('data/images/bus.jpg')[:, :, ::-1], # OpenCV
+ Image.open('data/images/bus.jpg'), # PIL
+ np.zeros((320, 640, 3))] # numpy
+
+ # Inference
+ results = model(imgs, size=320) # batched inference
+
+ # Results
+ results.print()
+ results.save()
diff --git a/logoAI.png b/logoAI.png
new file mode 100644
index 0000000000000000000000000000000000000000..dc5abc2859d20626e2582d01a3b2189e8ce5d28a
Binary files /dev/null and b/logoAI.png differ
diff --git a/models/__init__.py b/models/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..84952a8167bc2975913a6def6b4f027d566552a9
--- /dev/null
+++ b/models/__init__.py
@@ -0,0 +1 @@
+# init
\ No newline at end of file
diff --git a/models/__pycache__/__init__.cpython-310.pyc b/models/__pycache__/__init__.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..8e2050ccfc47e8f160b818a8673e595f9f804c5a
Binary files /dev/null and b/models/__pycache__/__init__.cpython-310.pyc differ
diff --git a/models/__pycache__/__init__.cpython-311.pyc b/models/__pycache__/__init__.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..464d277ac4b3526e22b1f5d9a48ad597845d0a79
Binary files /dev/null and b/models/__pycache__/__init__.cpython-311.pyc differ
diff --git a/models/__pycache__/common.cpython-310.pyc b/models/__pycache__/common.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..cec76008266053d13a7ca06f82960c29caac5a6c
Binary files /dev/null and b/models/__pycache__/common.cpython-310.pyc differ
diff --git a/models/__pycache__/common.cpython-311.pyc b/models/__pycache__/common.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..86d8e6d17435fe7b7c190345cb69b74d90bc9f69
Binary files /dev/null and b/models/__pycache__/common.cpython-311.pyc differ
diff --git a/models/__pycache__/experimental.cpython-310.pyc b/models/__pycache__/experimental.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..b091063584de94d7b41a615f3fee2a82c61ca74a
Binary files /dev/null and b/models/__pycache__/experimental.cpython-310.pyc differ
diff --git a/models/__pycache__/experimental.cpython-311.pyc b/models/__pycache__/experimental.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..2957be525db22a094b9246967187576560416c18
Binary files /dev/null and b/models/__pycache__/experimental.cpython-311.pyc differ
diff --git a/models/__pycache__/yolo.cpython-310.pyc b/models/__pycache__/yolo.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..00ed92ff92c9cbcc8663c90699beb218b3808b20
Binary files /dev/null and b/models/__pycache__/yolo.cpython-310.pyc differ
diff --git a/models/__pycache__/yolo.cpython-311.pyc b/models/__pycache__/yolo.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..f34dd20533a3fd707d33598f3dc1835adfe758ac
Binary files /dev/null and b/models/__pycache__/yolo.cpython-311.pyc differ
diff --git a/models/common.py b/models/common.py
new file mode 100644
index 0000000000000000000000000000000000000000..297641d2cbd229cdae5ef74ef67723633cf9f334
--- /dev/null
+++ b/models/common.py
@@ -0,0 +1,1280 @@
+import ast
+import contextlib
+import json
+import math
+import platform
+import warnings
+import zipfile
+from collections import OrderedDict, namedtuple
+from copy import copy
+from pathlib import Path
+from urllib.parse import urlparse
+
+from typing import Optional
+
+import cv2
+import numpy as np
+import pandas as pd
+import requests
+import torch
+import torch.nn as nn
+from IPython.display import display
+from PIL import Image
+from torch.cuda import amp
+
+from utils import TryExcept
+from utils.dataloaders import exif_transpose, letterbox
+from utils.general import (LOGGER, ROOT, Profile, check_requirements, check_suffix, check_version, colorstr,
+ increment_path, is_notebook, make_divisible, non_max_suppression, scale_boxes,
+ xywh2xyxy, xyxy2xywh, yaml_load)
+from utils.plots import Annotator, colors, save_one_box
+from utils.torch_utils import copy_attr, smart_inference_mode
+
+
+def autopad(k, p=None, d=1): # kernel, padding, dilation
+ # Pad to 'same' shape outputs
+ if d > 1:
+ k = d * (k - 1) + 1 if isinstance(k, int) else [d * (x - 1) + 1 for x in k] # actual kernel-size
+ if p is None:
+ p = k // 2 if isinstance(k, int) else [x // 2 for x in k] # auto-pad
+ return p
+
+
+class Conv(nn.Module):
+ # Standard convolution with args(ch_in, ch_out, kernel, stride, padding, groups, dilation, activation)
+ default_act = nn.SiLU() # default activation
+
+ def __init__(self, c1, c2, k=1, s=1, p=None, g=1, d=1, act=True):
+ super().__init__()
+ self.conv = nn.Conv2d(c1, c2, k, s, autopad(k, p, d), groups=g, dilation=d, bias=False)
+ self.bn = nn.BatchNorm2d(c2)
+ self.act = self.default_act if act is True else act if isinstance(act, nn.Module) else nn.Identity()
+
+ def forward(self, x):
+ return self.act(self.bn(self.conv(x)))
+
+ def forward_fuse(self, x):
+ return self.act(self.conv(x))
+
+
+class AConv(nn.Module):
+ def __init__(self, c1, c2): # ch_in, ch_out, shortcut, kernels, groups, expand
+ super().__init__()
+ self.cv1 = Conv(c1, c2, 3, 2, 1)
+
+ def forward(self, x):
+ x = torch.nn.functional.avg_pool2d(x, 2, 1, 0, False, True)
+ return self.cv1(x)
+
+
+class ADown(nn.Module):
+ def __init__(self, c1, c2): # ch_in, ch_out, shortcut, kernels, groups, expand
+ super().__init__()
+ self.c = c2 // 2
+ self.cv1 = Conv(c1 // 2, self.c, 3, 2, 1)
+ self.cv2 = Conv(c1 // 2, self.c, 1, 1, 0)
+
+ def forward(self, x):
+ x = torch.nn.functional.avg_pool2d(x, 2, 1, 0, False, True)
+ x1,x2 = x.chunk(2, 1)
+ x1 = self.cv1(x1)
+ x2 = torch.nn.functional.max_pool2d(x2, 3, 2, 1)
+ x2 = self.cv2(x2)
+ return torch.cat((x1, x2), 1)
+
+
+class RepConvN(nn.Module):
+ """RepConv is a basic rep-style block, including training and deploy status
+ This code is based on https://github.com/DingXiaoH/RepVGG/blob/main/repvgg.py
+ """
+ default_act = nn.SiLU() # default activation
+
+ def __init__(self, c1, c2, k=3, s=1, p=1, g=1, d=1, act=True, bn=False, deploy=False):
+ super().__init__()
+ assert k == 3 and p == 1
+ self.g = g
+ self.c1 = c1
+ self.c2 = c2
+ self.act = self.default_act if act is True else act if isinstance(act, nn.Module) else nn.Identity()
+
+ self.bn = None
+ self.conv1 = Conv(c1, c2, k, s, p=p, g=g, act=False)
+ self.conv2 = Conv(c1, c2, 1, s, p=(p - k // 2), g=g, act=False)
+
+ def forward_fuse(self, x):
+ """Forward process"""
+ return self.act(self.conv(x))
+
+ def forward(self, x):
+ """Forward process"""
+ id_out = 0 if self.bn is None else self.bn(x)
+ return self.act(self.conv1(x) + self.conv2(x) + id_out)
+
+ def get_equivalent_kernel_bias(self):
+ kernel3x3, bias3x3 = self._fuse_bn_tensor(self.conv1)
+ kernel1x1, bias1x1 = self._fuse_bn_tensor(self.conv2)
+ kernelid, biasid = self._fuse_bn_tensor(self.bn)
+ return kernel3x3 + self._pad_1x1_to_3x3_tensor(kernel1x1) + kernelid, bias3x3 + bias1x1 + biasid
+
+ def _avg_to_3x3_tensor(self, avgp):
+ channels = self.c1
+ groups = self.g
+ kernel_size = avgp.kernel_size
+ input_dim = channels // groups
+ k = torch.zeros((channels, input_dim, kernel_size, kernel_size))
+ k[np.arange(channels), np.tile(np.arange(input_dim), groups), :, :] = 1.0 / kernel_size ** 2
+ return k
+
+ def _pad_1x1_to_3x3_tensor(self, kernel1x1):
+ if kernel1x1 is None:
+ return 0
+ else:
+ return torch.nn.functional.pad(kernel1x1, [1, 1, 1, 1])
+
+ def _fuse_bn_tensor(self, branch):
+ if branch is None:
+ return 0, 0
+ if isinstance(branch, Conv):
+ kernel = branch.conv.weight
+ running_mean = branch.bn.running_mean
+ running_var = branch.bn.running_var
+ gamma = branch.bn.weight
+ beta = branch.bn.bias
+ eps = branch.bn.eps
+ elif isinstance(branch, nn.BatchNorm2d):
+ if not hasattr(self, 'id_tensor'):
+ input_dim = self.c1 // self.g
+ kernel_value = np.zeros((self.c1, input_dim, 3, 3), dtype=np.float32)
+ for i in range(self.c1):
+ kernel_value[i, i % input_dim, 1, 1] = 1
+ self.id_tensor = torch.from_numpy(kernel_value).to(branch.weight.device)
+ kernel = self.id_tensor
+ running_mean = branch.running_mean
+ running_var = branch.running_var
+ gamma = branch.weight
+ beta = branch.bias
+ eps = branch.eps
+ std = (running_var + eps).sqrt()
+ t = (gamma / std).reshape(-1, 1, 1, 1)
+ return kernel * t, beta - running_mean * gamma / std
+
+ def fuse_convs(self):
+ if hasattr(self, 'conv'):
+ return
+ kernel, bias = self.get_equivalent_kernel_bias()
+ self.conv = nn.Conv2d(in_channels=self.conv1.conv.in_channels,
+ out_channels=self.conv1.conv.out_channels,
+ kernel_size=self.conv1.conv.kernel_size,
+ stride=self.conv1.conv.stride,
+ padding=self.conv1.conv.padding,
+ dilation=self.conv1.conv.dilation,
+ groups=self.conv1.conv.groups,
+ bias=True).requires_grad_(False)
+ self.conv.weight.data = kernel
+ self.conv.bias.data = bias
+ for para in self.parameters():
+ para.detach_()
+ self.__delattr__('conv1')
+ self.__delattr__('conv2')
+ if hasattr(self, 'nm'):
+ self.__delattr__('nm')
+ if hasattr(self, 'bn'):
+ self.__delattr__('bn')
+ if hasattr(self, 'id_tensor'):
+ self.__delattr__('id_tensor')
+
+
+class SP(nn.Module):
+ def __init__(self, k=3, s=1):
+ super(SP, self).__init__()
+ self.m = nn.MaxPool2d(kernel_size=k, stride=s, padding=k // 2)
+
+ def forward(self, x):
+ return self.m(x)
+
+
+class MP(nn.Module):
+ # Max pooling
+ def __init__(self, k=2):
+ super(MP, self).__init__()
+ self.m = nn.MaxPool2d(kernel_size=k, stride=k)
+
+ def forward(self, x):
+ return self.m(x)
+
+
+class ConvTranspose(nn.Module):
+ # Convolution transpose 2d layer
+ default_act = nn.SiLU() # default activation
+
+ def __init__(self, c1, c2, k=2, s=2, p=0, bn=True, act=True):
+ super().__init__()
+ self.conv_transpose = nn.ConvTranspose2d(c1, c2, k, s, p, bias=not bn)
+ self.bn = nn.BatchNorm2d(c2) if bn else nn.Identity()
+ self.act = self.default_act if act is True else act if isinstance(act, nn.Module) else nn.Identity()
+
+ def forward(self, x):
+ return self.act(self.bn(self.conv_transpose(x)))
+
+
+class DWConv(Conv):
+ # Depth-wise convolution
+ def __init__(self, c1, c2, k=1, s=1, d=1, act=True): # ch_in, ch_out, kernel, stride, dilation, activation
+ super().__init__(c1, c2, k, s, g=math.gcd(c1, c2), d=d, act=act)
+
+
+class DWConvTranspose2d(nn.ConvTranspose2d):
+ # Depth-wise transpose convolution
+ def __init__(self, c1, c2, k=1, s=1, p1=0, p2=0): # ch_in, ch_out, kernel, stride, padding, padding_out
+ super().__init__(c1, c2, k, s, p1, p2, groups=math.gcd(c1, c2))
+
+
+class DFL(nn.Module):
+ # DFL module
+ def __init__(self, c1=17):
+ super().__init__()
+ self.conv = nn.Conv2d(c1, 1, 1, bias=False).requires_grad_(False)
+ self.conv.weight.data[:] = nn.Parameter(torch.arange(c1, dtype=torch.float).view(1, c1, 1, 1)) # / 120.0
+ self.c1 = c1
+ # self.bn = nn.BatchNorm2d(4)
+
+ def forward(self, x):
+ b, c, a = x.shape # batch, channels, anchors
+ return self.conv(x.view(b, 4, self.c1, a).transpose(2, 1).softmax(1)).view(b, 4, a)
+ # return self.conv(x.view(b, self.c1, 4, a).softmax(1)).view(b, 4, a)
+
+
+class BottleneckBase(nn.Module):
+ # Standard bottleneck
+ def __init__(self, c1, c2, shortcut=True, g=1, k=(1, 3), e=0.5): # ch_in, ch_out, shortcut, kernels, groups, expand
+ super().__init__()
+ c_ = int(c2 * e) # hidden channels
+ self.cv1 = Conv(c1, c_, k[0], 1)
+ self.cv2 = Conv(c_, c2, k[1], 1, g=g)
+ self.add = shortcut and c1 == c2
+
+ def forward(self, x):
+ return x + self.cv2(self.cv1(x)) if self.add else self.cv2(self.cv1(x))
+
+
+class RBottleneckBase(nn.Module):
+ # Standard bottleneck
+ def __init__(self, c1, c2, shortcut=True, g=1, k=(3, 1), e=0.5): # ch_in, ch_out, shortcut, kernels, groups, expand
+ super().__init__()
+ c_ = int(c2 * e) # hidden channels
+ self.cv1 = Conv(c1, c_, k[0], 1)
+ self.cv2 = Conv(c_, c2, k[1], 1, g=g)
+ self.add = shortcut and c1 == c2
+
+ def forward(self, x):
+ return x + self.cv2(self.cv1(x)) if self.add else self.cv2(self.cv1(x))
+
+
+class RepNRBottleneckBase(nn.Module):
+ # Standard bottleneck
+ def __init__(self, c1, c2, shortcut=True, g=1, k=(3, 1), e=0.5): # ch_in, ch_out, shortcut, kernels, groups, expand
+ super().__init__()
+ c_ = int(c2 * e) # hidden channels
+ self.cv1 = RepConvN(c1, c_, k[0], 1)
+ self.cv2 = Conv(c_, c2, k[1], 1, g=g)
+ self.add = shortcut and c1 == c2
+
+ def forward(self, x):
+ return x + self.cv2(self.cv1(x)) if self.add else self.cv2(self.cv1(x))
+
+
+class Bottleneck(nn.Module):
+ # Standard bottleneck
+ def __init__(self, c1, c2, shortcut=True, g=1, k=(3, 3), e=0.5): # ch_in, ch_out, shortcut, kernels, groups, expand
+ super().__init__()
+ c_ = int(c2 * e) # hidden channels
+ self.cv1 = Conv(c1, c_, k[0], 1)
+ self.cv2 = Conv(c_, c2, k[1], 1, g=g)
+ self.add = shortcut and c1 == c2
+
+ def forward(self, x):
+ return x + self.cv2(self.cv1(x)) if self.add else self.cv2(self.cv1(x))
+
+
+class RepNBottleneck(nn.Module):
+ # Standard bottleneck
+ def __init__(self, c1, c2, shortcut=True, g=1, k=(3, 3), e=0.5): # ch_in, ch_out, shortcut, kernels, groups, expand
+ super().__init__()
+ c_ = int(c2 * e) # hidden channels
+ self.cv1 = RepConvN(c1, c_, k[0], 1)
+ self.cv2 = Conv(c_, c2, k[1], 1, g=g)
+ self.add = shortcut and c1 == c2
+
+ def forward(self, x):
+ return x + self.cv2(self.cv1(x)) if self.add else self.cv2(self.cv1(x))
+
+
+class Res(nn.Module):
+ # ResNet bottleneck
+ def __init__(self, c1, c2, shortcut=True, g=1, e=0.5): # ch_in, ch_out, shortcut, groups, expansion
+ super(Res, self).__init__()
+ c_ = int(c2 * e) # hidden channels
+ self.cv1 = Conv(c1, c_, 1, 1)
+ self.cv2 = Conv(c_, c_, 3, 1, g=g)
+ self.cv3 = Conv(c_, c2, 1, 1)
+ self.add = shortcut and c1 == c2
+
+ def forward(self, x):
+ return x + self.cv3(self.cv2(self.cv1(x))) if self.add else self.cv3(self.cv2(self.cv1(x)))
+
+
+class RepNRes(nn.Module):
+ # ResNet bottleneck
+ def __init__(self, c1, c2, shortcut=True, g=1, e=0.5): # ch_in, ch_out, shortcut, groups, expansion
+ super(RepNRes, self).__init__()
+ c_ = int(c2 * e) # hidden channels
+ self.cv1 = Conv(c1, c_, 1, 1)
+ self.cv2 = RepConvN(c_, c_, 3, 1, g=g)
+ self.cv3 = Conv(c_, c2, 1, 1)
+ self.add = shortcut and c1 == c2
+
+ def forward(self, x):
+ return x + self.cv3(self.cv2(self.cv1(x))) if self.add else self.cv3(self.cv2(self.cv1(x)))
+
+
+class BottleneckCSP(nn.Module):
+ # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks
+ def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion
+ super().__init__()
+ c_ = int(c2 * e) # hidden channels
+ self.cv1 = Conv(c1, c_, 1, 1)
+ self.cv2 = nn.Conv2d(c1, c_, 1, 1, bias=False)
+ self.cv3 = nn.Conv2d(c_, c_, 1, 1, bias=False)
+ self.cv4 = Conv(2 * c_, c2, 1, 1)
+ self.bn = nn.BatchNorm2d(2 * c_) # applied to cat(cv2, cv3)
+ self.act = nn.SiLU()
+ self.m = nn.Sequential(*(Bottleneck(c_, c_, shortcut, g, e=1.0) for _ in range(n)))
+
+ def forward(self, x):
+ y1 = self.cv3(self.m(self.cv1(x)))
+ y2 = self.cv2(x)
+ return self.cv4(self.act(self.bn(torch.cat((y1, y2), 1))))
+
+
+class CSP(nn.Module):
+ # CSP Bottleneck with 3 convolutions
+ def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion
+ super().__init__()
+ c_ = int(c2 * e) # hidden channels
+ self.cv1 = Conv(c1, c_, 1, 1)
+ self.cv2 = Conv(c1, c_, 1, 1)
+ self.cv3 = Conv(2 * c_, c2, 1) # optional act=FReLU(c2)
+ self.m = nn.Sequential(*(Bottleneck(c_, c_, shortcut, g, e=1.0) for _ in range(n)))
+
+ def forward(self, x):
+ return self.cv3(torch.cat((self.m(self.cv1(x)), self.cv2(x)), 1))
+
+
+class RepNCSP(nn.Module):
+ # CSP Bottleneck with 3 convolutions
+ def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion
+ super().__init__()
+ c_ = int(c2 * e) # hidden channels
+ self.cv1 = Conv(c1, c_, 1, 1)
+ self.cv2 = Conv(c1, c_, 1, 1)
+ self.cv3 = Conv(2 * c_, c2, 1) # optional act=FReLU(c2)
+ self.m = nn.Sequential(*(RepNBottleneck(c_, c_, shortcut, g, e=1.0) for _ in range(n)))
+
+ def forward(self, x):
+ return self.cv3(torch.cat((self.m(self.cv1(x)), self.cv2(x)), 1))
+
+
+class CSPBase(nn.Module):
+ # CSP Bottleneck with 3 convolutions
+ def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion
+ super().__init__()
+ c_ = int(c2 * e) # hidden channels
+ self.cv1 = Conv(c1, c_, 1, 1)
+ self.cv2 = Conv(c1, c_, 1, 1)
+ self.cv3 = Conv(2 * c_, c2, 1) # optional act=FReLU(c2)
+ self.m = nn.Sequential(*(BottleneckBase(c_, c_, shortcut, g, e=1.0) for _ in range(n)))
+
+ def forward(self, x):
+ return self.cv3(torch.cat((self.m(self.cv1(x)), self.cv2(x)), 1))
+
+
+class SPP(nn.Module):
+ # Spatial Pyramid Pooling (SPP) layer https://arxiv.org/abs/1406.4729
+ def __init__(self, c1, c2, k=(5, 9, 13)):
+ super().__init__()
+ c_ = c1 // 2 # hidden channels
+ self.cv1 = Conv(c1, c_, 1, 1)
+ self.cv2 = Conv(c_ * (len(k) + 1), c2, 1, 1)
+ self.m = nn.ModuleList([nn.MaxPool2d(kernel_size=x, stride=1, padding=x // 2) for x in k])
+
+ def forward(self, x):
+ x = self.cv1(x)
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore') # suppress torch 1.9.0 max_pool2d() warning
+ return self.cv2(torch.cat([x] + [m(x) for m in self.m], 1))
+
+
+class ASPP(torch.nn.Module):
+
+ def __init__(self, in_channels, out_channels):
+ super().__init__()
+ kernel_sizes = [1, 3, 3, 1]
+ dilations = [1, 3, 6, 1]
+ paddings = [0, 3, 6, 0]
+ self.aspp = torch.nn.ModuleList()
+ for aspp_idx in range(len(kernel_sizes)):
+ conv = torch.nn.Conv2d(
+ in_channels,
+ out_channels,
+ kernel_size=kernel_sizes[aspp_idx],
+ stride=1,
+ dilation=dilations[aspp_idx],
+ padding=paddings[aspp_idx],
+ bias=True)
+ self.aspp.append(conv)
+ self.gap = torch.nn.AdaptiveAvgPool2d(1)
+ self.aspp_num = len(kernel_sizes)
+ for m in self.modules():
+ if isinstance(m, torch.nn.Conv2d):
+ n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
+ m.weight.data.normal_(0, math.sqrt(2. / n))
+ m.bias.data.fill_(0)
+
+ def forward(self, x):
+ avg_x = self.gap(x)
+ out = []
+ for aspp_idx in range(self.aspp_num):
+ inp = avg_x if (aspp_idx == self.aspp_num - 1) else x
+ out.append(F.relu_(self.aspp[aspp_idx](inp)))
+ out[-1] = out[-1].expand_as(out[-2])
+ out = torch.cat(out, dim=1)
+ return out
+
+
+class SPPCSPC(nn.Module):
+ # CSP SPP https://github.com/WongKinYiu/CrossStagePartialNetworks
+ def __init__(self, c1, c2, n=1, shortcut=False, g=1, e=0.5, k=(5, 9, 13)):
+ super(SPPCSPC, self).__init__()
+ c_ = int(2 * c2 * e) # hidden channels
+ self.cv1 = Conv(c1, c_, 1, 1)
+ self.cv2 = Conv(c1, c_, 1, 1)
+ self.cv3 = Conv(c_, c_, 3, 1)
+ self.cv4 = Conv(c_, c_, 1, 1)
+ self.m = nn.ModuleList([nn.MaxPool2d(kernel_size=x, stride=1, padding=x // 2) for x in k])
+ self.cv5 = Conv(4 * c_, c_, 1, 1)
+ self.cv6 = Conv(c_, c_, 3, 1)
+ self.cv7 = Conv(2 * c_, c2, 1, 1)
+
+ def forward(self, x):
+ x1 = self.cv4(self.cv3(self.cv1(x)))
+ y1 = self.cv6(self.cv5(torch.cat([x1] + [m(x1) for m in self.m], 1)))
+ y2 = self.cv2(x)
+ return self.cv7(torch.cat((y1, y2), dim=1))
+
+
+class SPPF(nn.Module):
+ # Spatial Pyramid Pooling - Fast (SPPF) layer by Glenn Jocher
+ def __init__(self, c1, c2, k=5): # equivalent to SPP(k=(5, 9, 13))
+ super().__init__()
+ c_ = c1 // 2 # hidden channels
+ self.cv1 = Conv(c1, c_, 1, 1)
+ self.cv2 = Conv(c_ * 4, c2, 1, 1)
+ self.m = nn.MaxPool2d(kernel_size=k, stride=1, padding=k // 2)
+ # self.m = SoftPool2d(kernel_size=k, stride=1, padding=k // 2)
+
+ def forward(self, x):
+ x = self.cv1(x)
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore') # suppress torch 1.9.0 max_pool2d() warning
+ y1 = self.m(x)
+ y2 = self.m(y1)
+ return self.cv2(torch.cat((x, y1, y2, self.m(y2)), 1))
+
+
+import torch.nn.functional as F
+from torch.nn.modules.utils import _pair
+
+
+class ReOrg(nn.Module):
+ # yolo
+ def __init__(self):
+ super(ReOrg, self).__init__()
+
+ def forward(self, x): # x(b,c,w,h) -> y(b,4c,w/2,h/2)
+ return torch.cat([x[..., ::2, ::2], x[..., 1::2, ::2], x[..., ::2, 1::2], x[..., 1::2, 1::2]], 1)
+
+
+class Contract(nn.Module):
+ # Contract width-height into channels, i.e. x(1,64,80,80) to x(1,256,40,40)
+ def __init__(self, gain=2):
+ super().__init__()
+ self.gain = gain
+
+ def forward(self, x):
+ b, c, h, w = x.size() # assert (h / s == 0) and (W / s == 0), 'Indivisible gain'
+ s = self.gain
+ x = x.view(b, c, h // s, s, w // s, s) # x(1,64,40,2,40,2)
+ x = x.permute(0, 3, 5, 1, 2, 4).contiguous() # x(1,2,2,64,40,40)
+ return x.view(b, c * s * s, h // s, w // s) # x(1,256,40,40)
+
+
+class Expand(nn.Module):
+ # Expand channels into width-height, i.e. x(1,64,80,80) to x(1,16,160,160)
+ def __init__(self, gain=2):
+ super().__init__()
+ self.gain = gain
+
+ def forward(self, x):
+ b, c, h, w = x.size() # assert C / s ** 2 == 0, 'Indivisible gain'
+ s = self.gain
+ x = x.view(b, s, s, c // s ** 2, h, w) # x(1,2,2,16,80,80)
+ x = x.permute(0, 3, 4, 1, 5, 2).contiguous() # x(1,16,80,2,80,2)
+ return x.view(b, c // s ** 2, h * s, w * s) # x(1,16,160,160)
+
+
+class Concat(nn.Module):
+ # Concatenate a list of tensors along dimension
+ def __init__(self, dimension=1):
+ super().__init__()
+ self.d = dimension
+
+ def forward(self, x):
+ return torch.cat(x, self.d)
+
+
+class Shortcut(nn.Module):
+ def __init__(self, dimension=0):
+ super(Shortcut, self).__init__()
+ self.d = dimension
+
+ def forward(self, x):
+ return x[0]+x[1]
+
+
+class Silence(nn.Module):
+ def __init__(self):
+ super(Silence, self).__init__()
+ def forward(self, x):
+ return x
+
+
+##### GELAN #####
+
+class SPPELAN(nn.Module):
+ # spp-elan
+ def __init__(self, c1, c2, c3): # ch_in, ch_out, number, shortcut, groups, expansion
+ super().__init__()
+ self.c = c3
+ self.cv1 = Conv(c1, c3, 1, 1)
+ self.cv2 = SP(5)
+ self.cv3 = SP(5)
+ self.cv4 = SP(5)
+ self.cv5 = Conv(4*c3, c2, 1, 1)
+
+ def forward(self, x):
+ y = [self.cv1(x)]
+ y.extend(m(y[-1]) for m in [self.cv2, self.cv3, self.cv4])
+ return self.cv5(torch.cat(y, 1))
+
+
+class ELAN1(nn.Module):
+
+ def __init__(self, c1, c2, c3, c4): # ch_in, ch_out, number, shortcut, groups, expansion
+ super().__init__()
+ self.c = c3//2
+ self.cv1 = Conv(c1, c3, 1, 1)
+ self.cv2 = Conv(c3//2, c4, 3, 1)
+ self.cv3 = Conv(c4, c4, 3, 1)
+ self.cv4 = Conv(c3+(2*c4), c2, 1, 1)
+
+ def forward(self, x):
+ y = list(self.cv1(x).chunk(2, 1))
+ y.extend(m(y[-1]) for m in [self.cv2, self.cv3])
+ return self.cv4(torch.cat(y, 1))
+
+ def forward_split(self, x):
+ y = list(self.cv1(x).split((self.c, self.c), 1))
+ y.extend(m(y[-1]) for m in [self.cv2, self.cv3])
+ return self.cv4(torch.cat(y, 1))
+
+
+class RepNCSPELAN4(nn.Module):
+ # csp-elan
+ def __init__(self, c1, c2, c3, c4, c5=1): # ch_in, ch_out, number, shortcut, groups, expansion
+ super().__init__()
+ self.c = c3//2
+ self.cv1 = Conv(c1, c3, 1, 1)
+ self.cv2 = nn.Sequential(RepNCSP(c3//2, c4, c5), Conv(c4, c4, 3, 1))
+ self.cv3 = nn.Sequential(RepNCSP(c4, c4, c5), Conv(c4, c4, 3, 1))
+ self.cv4 = Conv(c3+(2*c4), c2, 1, 1)
+
+ def forward(self, x):
+ y = list(self.cv1(x).chunk(2, 1))
+ y.extend((m(y[-1])) for m in [self.cv2, self.cv3])
+ return self.cv4(torch.cat(y, 1))
+
+ def forward_split(self, x):
+ y = list(self.cv1(x).split((self.c, self.c), 1))
+ y.extend(m(y[-1]) for m in [self.cv2, self.cv3])
+ return self.cv4(torch.cat(y, 1))
+
+#################
+#####add new model####
+class Attention(nn.Module):
+ def __init__(self, dim, num_heads=8,
+ attn_ratio=0.5):
+ super().__init__()
+ self.num_heads = num_heads
+ self.head_dim = dim // num_heads
+ self.key_dim = int(self.head_dim * attn_ratio)
+ self.scale = self.key_dim ** -0.5
+ nh_kd = nh_kd = self.key_dim * num_heads
+ h = dim + nh_kd * 2
+ self.qkv = Conv(dim, h, 1, act=False)
+ self.proj = Conv(dim, dim, 1, act=False)
+ self.pe = Conv(dim, dim, 3, 1, g=dim, act=False)
+
+ def forward(self, x):
+ B, C, H, W = x.shape
+ N = H * W
+ qkv = self.qkv(x)
+ q, k, v = qkv.view(B, self.num_heads, self.key_dim*2 + self.head_dim, N).split([self.key_dim, self.key_dim, self.head_dim], dim=2)
+
+ attn = (
+ (q.transpose(-2, -1) @ k) * self.scale
+ )
+ attn = attn.softmax(dim=-1)
+ x = (v @ attn.transpose(-2, -1)).view(B, C, H, W) + self.pe(v.reshape(B, C, H, W))
+ x = self.proj(x)
+ return x
+
+class PSA(nn.Module):
+
+ def __init__(self, c1, c2, e=0.5):
+ super().__init__()
+ assert(c1 == c2)
+ self.c = int(c1 * e)
+ self.cv1 = Conv(c1, 2 * self.c, 1, 1)
+ self.cv2 = Conv(2 * self.c, c1, 1)
+
+ self.attn = Attention(self.c, attn_ratio=0.5, num_heads=self.c // 64)
+ self.ffn = nn.Sequential(
+ Conv(self.c, self.c*2, 1),
+ Conv(self.c*2, self.c, 1, act=False)
+ )
+ def forward(self, x):
+ a, b = self.cv1(x).split((self.c, self.c), dim=1)
+ b = b + self.attn(b)
+ b = b + self.ffn(b)
+ return self.cv2(torch.cat((a, b), 1))
+
+##### YOLOR #####
+
+class ImplicitA(nn.Module):
+ def __init__(self, channel):
+ super(ImplicitA, self).__init__()
+ self.channel = channel
+ self.implicit = nn.Parameter(torch.zeros(1, channel, 1, 1))
+ nn.init.normal_(self.implicit, std=.02)
+
+ def forward(self, x):
+ return self.implicit + x
+
+
+class ImplicitM(nn.Module):
+ def __init__(self, channel):
+ super(ImplicitM, self).__init__()
+ self.channel = channel
+ self.implicit = nn.Parameter(torch.ones(1, channel, 1, 1))
+ nn.init.normal_(self.implicit, mean=1., std=.02)
+
+ def forward(self, x):
+ return self.implicit * x
+
+#################
+
+
+##### CBNet #####
+
+class CBLinear(nn.Module):
+ def __init__(self, c1, c2s, k=1, s=1, p=None, g=1): # ch_in, ch_outs, kernel, stride, padding, groups
+ super(CBLinear, self).__init__()
+ self.c2s = c2s
+ self.conv = nn.Conv2d(c1, sum(c2s), k, s, autopad(k, p), groups=g, bias=True)
+
+ def forward(self, x):
+ outs = self.conv(x).split(self.c2s, dim=1)
+ return outs
+
+class CBFuse(nn.Module):
+ def __init__(self, idx):
+ super(CBFuse, self).__init__()
+ self.idx = idx
+
+ def forward(self, xs):
+ target_size = xs[-1].shape[2:]
+ res = [F.interpolate(x[self.idx[i]], size=target_size, mode='nearest') for i, x in enumerate(xs[:-1])]
+ out = torch.sum(torch.stack(res + xs[-1:]), dim=0)
+ return out
+
+#################
+
+
+class DetectMultiBackend(nn.Module):
+ # YOLO MultiBackend class for python inference on various backends
+ def __init__(self, weights='yolo.pt', device=torch.device('cpu'), dnn=False, data=None, fp16=False, fuse=True):
+ # Usage:
+ # PyTorch: weights = *.pt
+ # TorchScript: *.torchscript
+ # ONNX Runtime: *.onnx
+ # ONNX OpenCV DNN: *.onnx --dnn
+ # OpenVINO: *_openvino_model
+ # CoreML: *.mlmodel
+ # TensorRT: *.engine
+ # TensorFlow SavedModel: *_saved_model
+ # TensorFlow GraphDef: *.pb
+ # TensorFlow Lite: *.tflite
+ # TensorFlow Edge TPU: *_edgetpu.tflite
+ # PaddlePaddle: *_paddle_model
+ from models.experimental import attempt_download, attempt_load # scoped to avoid circular import
+
+ super().__init__()
+ w = str(weights[0] if isinstance(weights, list) else weights)
+ pt, jit, onnx, onnx_end2end, xml, engine, coreml, saved_model, pb, tflite, edgetpu, tfjs, paddle, triton = self._model_type(w)
+ fp16 &= pt or jit or onnx or engine # FP16
+ nhwc = coreml or saved_model or pb or tflite or edgetpu # BHWC formats (vs torch BCWH)
+ stride = 32 # default stride
+ cuda = torch.cuda.is_available() and device.type != 'cpu' # use CUDA
+ if not (pt or triton):
+ w = attempt_download(w) # download if not local
+
+ if pt: # PyTorch
+ model = attempt_load(weights if isinstance(weights, list) else w, device=device, inplace=True, fuse=fuse)
+ stride = max(int(model.stride.max()), 32) # model stride
+ names = model.module.names if hasattr(model, 'module') else model.names # get class names
+ model.half() if fp16 else model.float()
+ self.model = model # explicitly assign for to(), cpu(), cuda(), half()
+ elif jit: # TorchScript
+ LOGGER.info(f'Loading {w} for TorchScript inference...')
+ extra_files = {'config.txt': ''} # model metadata
+ model = torch.jit.load(w, _extra_files=extra_files, map_location=device)
+ model.half() if fp16 else model.float()
+ if extra_files['config.txt']: # load metadata dict
+ d = json.loads(extra_files['config.txt'],
+ object_hook=lambda d: {int(k) if k.isdigit() else k: v
+ for k, v in d.items()})
+ stride, names = int(d['stride']), d['names']
+ elif dnn: # ONNX OpenCV DNN
+ LOGGER.info(f'Loading {w} for ONNX OpenCV DNN inference...')
+ check_requirements('opencv-python>=4.5.4')
+ net = cv2.dnn.readNetFromONNX(w)
+ elif onnx: # ONNX Runtime
+ LOGGER.info(f'Loading {w} for ONNX Runtime inference...')
+ check_requirements(('onnx', 'onnxruntime-gpu' if cuda else 'onnxruntime'))
+ import onnxruntime
+ providers = ['CUDAExecutionProvider', 'CPUExecutionProvider'] if cuda else ['CPUExecutionProvider']
+ session = onnxruntime.InferenceSession(w, providers=providers)
+ output_names = [x.name for x in session.get_outputs()]
+ meta = session.get_modelmeta().custom_metadata_map # metadata
+ if 'stride' in meta:
+ stride, names = int(meta['stride']), eval(meta['names'])
+ elif xml: # OpenVINO
+ LOGGER.info(f'Loading {w} for OpenVINO inference...')
+ check_requirements('openvino') # requires openvino-dev: https://pypi.org/project/openvino-dev/
+ from openvino.runtime import Core, Layout, get_batch
+ ie = Core()
+ if not Path(w).is_file(): # if not *.xml
+ w = next(Path(w).glob('*.xml')) # get *.xml file from *_openvino_model dir
+ network = ie.read_model(model=w, weights=Path(w).with_suffix('.bin'))
+ if network.get_parameters()[0].get_layout().empty:
+ network.get_parameters()[0].set_layout(Layout("NCHW"))
+ batch_dim = get_batch(network)
+ if batch_dim.is_static:
+ batch_size = batch_dim.get_length()
+ executable_network = ie.compile_model(network, device_name="CPU") # device_name="MYRIAD" for Intel NCS2
+ stride, names = self._load_metadata(Path(w).with_suffix('.yaml')) # load metadata
+ elif engine: # TensorRT
+ LOGGER.info(f'Loading {w} for TensorRT inference...')
+ import tensorrt as trt # https://developer.nvidia.com/nvidia-tensorrt-download
+ check_version(trt.__version__, '7.0.0', hard=True) # require tensorrt>=7.0.0
+ if device.type == 'cpu':
+ device = torch.device('cuda:0')
+ Binding = namedtuple('Binding', ('name', 'dtype', 'shape', 'data', 'ptr'))
+ logger = trt.Logger(trt.Logger.INFO)
+ with open(w, 'rb') as f, trt.Runtime(logger) as runtime:
+ model = runtime.deserialize_cuda_engine(f.read())
+ context = model.create_execution_context()
+ bindings = OrderedDict()
+ output_names = []
+ fp16 = False # default updated below
+ dynamic = False
+ for i in range(model.num_bindings):
+ name = model.get_binding_name(i)
+ dtype = trt.nptype(model.get_binding_dtype(i))
+ if model.binding_is_input(i):
+ if -1 in tuple(model.get_binding_shape(i)): # dynamic
+ dynamic = True
+ context.set_binding_shape(i, tuple(model.get_profile_shape(0, i)[2]))
+ if dtype == np.float16:
+ fp16 = True
+ else: # output
+ output_names.append(name)
+ shape = tuple(context.get_binding_shape(i))
+ im = torch.from_numpy(np.empty(shape, dtype=dtype)).to(device)
+ bindings[name] = Binding(name, dtype, shape, im, int(im.data_ptr()))
+ binding_addrs = OrderedDict((n, d.ptr) for n, d in bindings.items())
+ batch_size = bindings['images'].shape[0] # if dynamic, this is instead max batch size
+ elif coreml: # CoreML
+ LOGGER.info(f'Loading {w} for CoreML inference...')
+ import coremltools as ct
+ model = ct.models.MLModel(w)
+ elif saved_model: # TF SavedModel
+ LOGGER.info(f'Loading {w} for TensorFlow SavedModel inference...')
+ import tensorflow as tf
+ keras = False # assume TF1 saved_model
+ model = tf.keras.models.load_model(w) if keras else tf.saved_model.load(w)
+ elif pb: # GraphDef https://www.tensorflow.org/guide/migrate#a_graphpb_or_graphpbtxt
+ LOGGER.info(f'Loading {w} for TensorFlow GraphDef inference...')
+ import tensorflow as tf
+
+ def wrap_frozen_graph(gd, inputs, outputs):
+ x = tf.compat.v1.wrap_function(lambda: tf.compat.v1.import_graph_def(gd, name=""), []) # wrapped
+ ge = x.graph.as_graph_element
+ return x.prune(tf.nest.map_structure(ge, inputs), tf.nest.map_structure(ge, outputs))
+
+ def gd_outputs(gd):
+ name_list, input_list = [], []
+ for node in gd.node: # tensorflow.core.framework.node_def_pb2.NodeDef
+ name_list.append(node.name)
+ input_list.extend(node.input)
+ return sorted(f'{x}:0' for x in list(set(name_list) - set(input_list)) if not x.startswith('NoOp'))
+
+ gd = tf.Graph().as_graph_def() # TF GraphDef
+ with open(w, 'rb') as f:
+ gd.ParseFromString(f.read())
+ frozen_func = wrap_frozen_graph(gd, inputs="x:0", outputs=gd_outputs(gd))
+ elif tflite or edgetpu: # https://www.tensorflow.org/lite/guide/python#install_tensorflow_lite_for_python
+ try: # https://coral.ai/docs/edgetpu/tflite-python/#update-existing-tf-lite-code-for-the-edge-tpu
+ from tflite_runtime.interpreter import Interpreter, load_delegate
+ except ImportError:
+ import tensorflow as tf
+ Interpreter, load_delegate = tf.lite.Interpreter, tf.lite.experimental.load_delegate,
+ if edgetpu: # TF Edge TPU https://coral.ai/software/#edgetpu-runtime
+ LOGGER.info(f'Loading {w} for TensorFlow Lite Edge TPU inference...')
+ delegate = {
+ 'Linux': 'libedgetpu.so.1',
+ 'Darwin': 'libedgetpu.1.dylib',
+ 'Windows': 'edgetpu.dll'}[platform.system()]
+ interpreter = Interpreter(model_path=w, experimental_delegates=[load_delegate(delegate)])
+ else: # TFLite
+ LOGGER.info(f'Loading {w} for TensorFlow Lite inference...')
+ interpreter = Interpreter(model_path=w) # load TFLite model
+ interpreter.allocate_tensors() # allocate
+ input_details = interpreter.get_input_details() # inputs
+ output_details = interpreter.get_output_details() # outputs
+ # load metadata
+ with contextlib.suppress(zipfile.BadZipFile):
+ with zipfile.ZipFile(w, "r") as model:
+ meta_file = model.namelist()[0]
+ meta = ast.literal_eval(model.read(meta_file).decode("utf-8"))
+ stride, names = int(meta['stride']), meta['names']
+ elif tfjs: # TF.js
+ raise NotImplementedError('ERROR: YOLO TF.js inference is not supported')
+ elif paddle: # PaddlePaddle
+ LOGGER.info(f'Loading {w} for PaddlePaddle inference...')
+ check_requirements('paddlepaddle-gpu' if cuda else 'paddlepaddle')
+ import paddle.inference as pdi
+ if not Path(w).is_file(): # if not *.pdmodel
+ w = next(Path(w).rglob('*.pdmodel')) # get *.pdmodel file from *_paddle_model dir
+ weights = Path(w).with_suffix('.pdiparams')
+ config = pdi.Config(str(w), str(weights))
+ if cuda:
+ config.enable_use_gpu(memory_pool_init_size_mb=2048, device_id=0)
+ predictor = pdi.create_predictor(config)
+ input_handle = predictor.get_input_handle(predictor.get_input_names()[0])
+ output_names = predictor.get_output_names()
+ elif triton: # NVIDIA Triton Inference Server
+ LOGGER.info(f'Using {w} as Triton Inference Server...')
+ check_requirements('tritonclient[all]')
+ from utils.triton import TritonRemoteModel
+ model = TritonRemoteModel(url=w)
+ nhwc = model.runtime.startswith("tensorflow")
+ else:
+ raise NotImplementedError(f'ERROR: {w} is not a supported format')
+
+ # class names
+ if 'names' not in locals():
+ names = yaml_load(data)['names'] if data else {i: f'class{i}' for i in range(999)}
+ if names[0] == 'n01440764' and len(names) == 1000: # ImageNet
+ names = yaml_load(ROOT / 'data/ImageNet.yaml')['names'] # human-readable names
+
+ self.__dict__.update(locals()) # assign all variables to self
+
+ def forward(self, im, augment=False, visualize=False):
+ # YOLO MultiBackend inference
+ b, ch, h, w = im.shape # batch, channel, height, width
+ if self.fp16 and im.dtype != torch.float16:
+ im = im.half() # to FP16
+ if self.nhwc:
+ im = im.permute(0, 2, 3, 1) # torch BCHW to numpy BHWC shape(1,320,192,3)
+
+ if self.pt: # PyTorch
+ y = self.model(im, augment=augment, visualize=visualize) if augment or visualize else self.model(im)
+ elif self.jit: # TorchScript
+ y = self.model(im)
+ elif self.dnn: # ONNX OpenCV DNN
+ im = im.cpu().numpy() # torch to numpy
+ self.net.setInput(im)
+ y = self.net.forward()
+ elif self.onnx: # ONNX Runtime
+ im = im.cpu().numpy() # torch to numpy
+ y = self.session.run(self.output_names, {self.session.get_inputs()[0].name: im})
+ elif self.xml: # OpenVINO
+ im = im.cpu().numpy() # FP32
+ y = list(self.executable_network([im]).values())
+ elif self.engine: # TensorRT
+ if self.dynamic and im.shape != self.bindings['images'].shape:
+ i = self.model.get_binding_index('images')
+ self.context.set_binding_shape(i, im.shape) # reshape if dynamic
+ self.bindings['images'] = self.bindings['images']._replace(shape=im.shape)
+ for name in self.output_names:
+ i = self.model.get_binding_index(name)
+ self.bindings[name].data.resize_(tuple(self.context.get_binding_shape(i)))
+ s = self.bindings['images'].shape
+ assert im.shape == s, f"input size {im.shape} {'>' if self.dynamic else 'not equal to'} max model size {s}"
+ self.binding_addrs['images'] = int(im.data_ptr())
+ self.context.execute_v2(list(self.binding_addrs.values()))
+ y = [self.bindings[x].data for x in sorted(self.output_names)]
+ elif self.coreml: # CoreML
+ im = im.cpu().numpy()
+ im = Image.fromarray((im[0] * 255).astype('uint8'))
+ # im = im.resize((192, 320), Image.ANTIALIAS)
+ y = self.model.predict({'image': im}) # coordinates are xywh normalized
+ if 'confidence' in y:
+ box = xywh2xyxy(y['coordinates'] * [[w, h, w, h]]) # xyxy pixels
+ conf, cls = y['confidence'].max(1), y['confidence'].argmax(1).astype(np.float)
+ y = np.concatenate((box, conf.reshape(-1, 1), cls.reshape(-1, 1)), 1)
+ else:
+ y = list(reversed(y.values())) # reversed for segmentation models (pred, proto)
+ elif self.paddle: # PaddlePaddle
+ im = im.cpu().numpy().astype(np.float32)
+ self.input_handle.copy_from_cpu(im)
+ self.predictor.run()
+ y = [self.predictor.get_output_handle(x).copy_to_cpu() for x in self.output_names]
+ elif self.triton: # NVIDIA Triton Inference Server
+ y = self.model(im)
+ else: # TensorFlow (SavedModel, GraphDef, Lite, Edge TPU)
+ im = im.cpu().numpy()
+ if self.saved_model: # SavedModel
+ y = self.model(im, training=False) if self.keras else self.model(im)
+ elif self.pb: # GraphDef
+ y = self.frozen_func(x=self.tf.constant(im))
+ else: # Lite or Edge TPU
+ input = self.input_details[0]
+ int8 = input['dtype'] == np.uint8 # is TFLite quantized uint8 model
+ if int8:
+ scale, zero_point = input['quantization']
+ im = (im / scale + zero_point).astype(np.uint8) # de-scale
+ self.interpreter.set_tensor(input['index'], im)
+ self.interpreter.invoke()
+ y = []
+ for output in self.output_details:
+ x = self.interpreter.get_tensor(output['index'])
+ if int8:
+ scale, zero_point = output['quantization']
+ x = (x.astype(np.float32) - zero_point) * scale # re-scale
+ y.append(x)
+ y = [x if isinstance(x, np.ndarray) else x.numpy() for x in y]
+ y[0][..., :4] *= [w, h, w, h] # xywh normalized to pixels
+
+ if isinstance(y, (list, tuple)):
+ return self.from_numpy(y[0]) if len(y) == 1 else [self.from_numpy(x) for x in y]
+ else:
+ return self.from_numpy(y)
+
+ def from_numpy(self, x):
+ return torch.from_numpy(x).to(self.device) if isinstance(x, np.ndarray) else x
+
+ def warmup(self, imgsz=(1, 3, 640, 640)):
+ # Warmup model by running inference once
+ warmup_types = self.pt, self.jit, self.onnx, self.engine, self.saved_model, self.pb, self.triton
+ if any(warmup_types) and (self.device.type != 'cpu' or self.triton):
+ im = torch.empty(*imgsz, dtype=torch.half if self.fp16 else torch.float, device=self.device) # input
+ for _ in range(2 if self.jit else 1): #
+ self.forward(im) # warmup
+
+ @staticmethod
+ def _model_type(p='path/to/model.pt'):
+ # Return model type from model path, i.e. path='path/to/model.onnx' -> type=onnx
+ # types = [pt, jit, onnx, xml, engine, coreml, saved_model, pb, tflite, edgetpu, tfjs, paddle]
+ from export import export_formats
+ from utils.downloads import is_url
+ sf = list(export_formats().Suffix) # export suffixes
+ if not is_url(p, check=False):
+ check_suffix(p, sf) # checks
+ url = urlparse(p) # if url may be Triton inference server
+ types = [s in Path(p).name for s in sf]
+ types[8] &= not types[9] # tflite &= not edgetpu
+ triton = not any(types) and all([any(s in url.scheme for s in ["http", "grpc"]), url.netloc])
+ return types + [triton]
+
+ @staticmethod
+ def _load_metadata(f=Path('path/to/meta.yaml')):
+ # Load metadata from meta.yaml if it exists
+ if f.exists():
+ d = yaml_load(f)
+ return d['stride'], d['names'] # assign stride, names
+ return None, None
+
+
+class AutoShape(nn.Module):
+ # YOLO input-robust model wrapper for passing cv2/np/PIL/torch inputs. Includes preprocessing, inference and NMS
+ conf = 0.25 # NMS confidence threshold
+ iou = 0.45 # NMS IoU threshold
+ agnostic = False # NMS class-agnostic
+ multi_label = False # NMS multiple labels per box
+ classes = None # (optional list) filter by class, i.e. = [0, 15, 16] for COCO persons, cats and dogs
+ max_det = 1000 # maximum number of detections per image
+ amp = False # Automatic Mixed Precision (AMP) inference
+
+ def __init__(self, model, verbose=True):
+ super().__init__()
+ if verbose:
+ LOGGER.info('Adding AutoShape... ')
+ copy_attr(self, model, include=('yaml', 'nc', 'hyp', 'names', 'stride', 'abc'), exclude=()) # copy attributes
+ self.dmb = isinstance(model, DetectMultiBackend) # DetectMultiBackend() instance
+ self.pt = not self.dmb or model.pt # PyTorch model
+ self.model = model.eval()
+ if self.pt:
+ m = self.model.model.model[-1] if self.dmb else self.model.model[-1] # Detect()
+ m.inplace = False # Detect.inplace=False for safe multithread inference
+ m.export = True # do not output loss values
+
+ def _apply(self, fn):
+ # Apply to(), cpu(), cuda(), half() to model tensors that are not parameters or registered buffers
+ self = super()._apply(fn)
+ from models.yolo import Detect, Segment
+ if self.pt:
+ m = self.model.model.model[-1] if self.dmb else self.model.model[-1] # Detect()
+ if isinstance(m, (Detect, Segment)):
+ for k in 'stride', 'anchor_grid', 'stride_grid', 'grid':
+ x = getattr(m, k)
+ setattr(m, k, list(map(fn, x))) if isinstance(x, (list, tuple)) else setattr(m, k, fn(x))
+ return self
+
+ @smart_inference_mode()
+ def forward(self, ims, size=640, augment=False, profile=False):
+ # Inference from various sources. For size(height=640, width=1280), RGB images example inputs are:
+ # file: ims = 'data/images/zidane.jpg' # str or PosixPath
+ # URI: = 'https://ultralytics.com/images/zidane.jpg'
+ # OpenCV: = cv2.imread('image.jpg')[:,:,::-1] # HWC BGR to RGB x(640,1280,3)
+ # PIL: = Image.open('image.jpg') or ImageGrab.grab() # HWC x(640,1280,3)
+ # numpy: = np.zeros((640,1280,3)) # HWC
+ # torch: = torch.zeros(16,3,320,640) # BCHW (scaled to size=640, 0-1 values)
+ # multiple: = [Image.open('image1.jpg'), Image.open('image2.jpg'), ...] # list of images
+
+ dt = (Profile(), Profile(), Profile())
+ with dt[0]:
+ if isinstance(size, int): # expand
+ size = (size, size)
+ p = next(self.model.parameters()) if self.pt else torch.empty(1, device=self.model.device) # param
+ autocast = self.amp and (p.device.type != 'cpu') # Automatic Mixed Precision (AMP) inference
+ if isinstance(ims, torch.Tensor): # torch
+ with amp.autocast(autocast):
+ return self.model(ims.to(p.device).type_as(p), augment=augment) # inference
+
+ # Pre-process
+ n, ims = (len(ims), list(ims)) if isinstance(ims, (list, tuple)) else (1, [ims]) # number, list of images
+ shape0, shape1, files = [], [], [] # image and inference shapes, filenames
+ for i, im in enumerate(ims):
+ f = f'image{i}' # filename
+ if isinstance(im, (str, Path)): # filename or uri
+ im, f = Image.open(requests.get(im, stream=True).raw if str(im).startswith('http') else im), im
+ im = np.asarray(exif_transpose(im))
+ elif isinstance(im, Image.Image): # PIL Image
+ im, f = np.asarray(exif_transpose(im)), getattr(im, 'filename', f) or f
+ files.append(Path(f).with_suffix('.jpg').name)
+ if im.shape[0] < 5: # image in CHW
+ im = im.transpose((1, 2, 0)) # reverse dataloader .transpose(2, 0, 1)
+ im = im[..., :3] if im.ndim == 3 else cv2.cvtColor(im, cv2.COLOR_GRAY2BGR) # enforce 3ch input
+ s = im.shape[:2] # HWC
+ shape0.append(s) # image shape
+ g = max(size) / max(s) # gain
+ shape1.append([int(y * g) for y in s])
+ ims[i] = im if im.data.contiguous else np.ascontiguousarray(im) # update
+ shape1 = [make_divisible(x, self.stride) for x in np.array(shape1).max(0)] # inf shape
+ x = [letterbox(im, shape1, auto=False)[0] for im in ims] # pad
+ x = np.ascontiguousarray(np.array(x).transpose((0, 3, 1, 2))) # stack and BHWC to BCHW
+ x = torch.from_numpy(x).to(p.device).type_as(p) / 255 # uint8 to fp16/32
+
+ with amp.autocast(autocast):
+ # Inference
+ with dt[1]:
+ y = self.model(x, augment=augment) # forward
+
+ # Post-process
+ with dt[2]:
+ y = non_max_suppression(y if self.dmb else y[0],
+ self.conf,
+ self.iou,
+ self.classes,
+ self.agnostic,
+ self.multi_label,
+ max_det=self.max_det) # NMS
+ for i in range(n):
+ scale_boxes(shape1, y[i][:, :4], shape0[i])
+
+ return Detections(ims, y, files, dt, self.names, x.shape)
+
+
+class Detections:
+ # YOLO detections class for inference results
+ def __init__(self, ims, pred, files, times=(0, 0, 0), names=None, shape=None):
+ super().__init__()
+ d = pred[0].device # device
+ gn = [torch.tensor([*(im.shape[i] for i in [1, 0, 1, 0]), 1, 1], device=d) for im in ims] # normalizations
+ self.ims = ims # list of images as numpy arrays
+ self.pred = pred # list of tensors pred[0] = (xyxy, conf, cls)
+ self.names = names # class names
+ self.files = files # image filenames
+ self.times = times # profiling times
+ self.xyxy = pred # xyxy pixels
+ self.xywh = [xyxy2xywh(x) for x in pred] # xywh pixels
+ self.xyxyn = [x / g for x, g in zip(self.xyxy, gn)] # xyxy normalized
+ self.xywhn = [x / g for x, g in zip(self.xywh, gn)] # xywh normalized
+ self.n = len(self.pred) # number of images (batch size)
+ self.t = tuple(x.t / self.n * 1E3 for x in times) # timestamps (ms)
+ self.s = tuple(shape) # inference BCHW shape
+
+ def _run(self, pprint=False, show=False, save=False, crop=False, render=False, labels=True, save_dir=Path('')):
+ s, crops = '', []
+ for i, (im, pred) in enumerate(zip(self.ims, self.pred)):
+ s += f'\nimage {i + 1}/{len(self.pred)}: {im.shape[0]}x{im.shape[1]} ' # string
+ if pred.shape[0]:
+ for c in pred[:, -1].unique():
+ n = (pred[:, -1] == c).sum() # detections per class
+ s += f"{n} {self.names[int(c)]}{'s' * (n > 1)}, " # add to string
+ s = s.rstrip(', ')
+ if show or save or render or crop:
+ annotator = Annotator(im, example=str(self.names))
+ for *box, conf, cls in reversed(pred): # xyxy, confidence, class
+ label = f'{self.names[int(cls)]} {conf:.2f}'
+ if crop:
+ file = save_dir / 'crops' / self.names[int(cls)] / self.files[i] if save else None
+ crops.append({
+ 'box': box,
+ 'conf': conf,
+ 'cls': cls,
+ 'label': label,
+ 'im': save_one_box(box, im, file=file, save=save)})
+ else: # all others
+ annotator.box_label(box, label if labels else '', color=colors(cls))
+ im = annotator.im
+ else:
+ s += '(no detections)'
+
+ im = Image.fromarray(im.astype(np.uint8)) if isinstance(im, np.ndarray) else im # from np
+ if show:
+ display(im) if is_notebook() else im.show(self.files[i])
+ if save:
+ f = self.files[i]
+ im.save(save_dir / f) # save
+ if i == self.n - 1:
+ LOGGER.info(f"Saved {self.n} image{'s' * (self.n > 1)} to {colorstr('bold', save_dir)}")
+ if render:
+ self.ims[i] = np.asarray(im)
+ if pprint:
+ s = s.lstrip('\n')
+ return f'{s}\nSpeed: %.1fms pre-process, %.1fms inference, %.1fms NMS per image at shape {self.s}' % self.t
+ if crop:
+ if save:
+ LOGGER.info(f'Saved results to {save_dir}\n')
+ return crops
+
+ @TryExcept('Showing images is not supported in this environment')
+ def show(self, labels=True):
+ self._run(show=True, labels=labels) # show results
+
+ def save(self, labels=True, save_dir='runs/detect/exp', exist_ok=False):
+ save_dir = increment_path(save_dir, exist_ok, mkdir=True) # increment save_dir
+ self._run(save=True, labels=labels, save_dir=save_dir) # save results
+
+ def crop(self, save=True, save_dir='runs/detect/exp', exist_ok=False):
+ save_dir = increment_path(save_dir, exist_ok, mkdir=True) if save else None
+ return self._run(crop=True, save=save, save_dir=save_dir) # crop results
+
+ def render(self, labels=True):
+ self._run(render=True, labels=labels) # render results
+ return self.ims
+
+ def pandas(self):
+ # return detections as pandas DataFrames, i.e. print(results.pandas().xyxy[0])
+ new = copy(self) # return copy
+ ca = 'xmin', 'ymin', 'xmax', 'ymax', 'confidence', 'class', 'name' # xyxy columns
+ cb = 'xcenter', 'ycenter', 'width', 'height', 'confidence', 'class', 'name' # xywh columns
+ for k, c in zip(['xyxy', 'xyxyn', 'xywh', 'xywhn'], [ca, ca, cb, cb]):
+ a = [[x[:5] + [int(x[5]), self.names[int(x[5])]] for x in x.tolist()] for x in getattr(self, k)] # update
+ setattr(new, k, [pd.DataFrame(x, columns=c) for x in a])
+ return new
+
+ def tolist(self):
+ # return a list of Detections objects, i.e. 'for result in results.tolist():'
+ r = range(self.n) # iterable
+ x = [Detections([self.ims[i]], [self.pred[i]], [self.files[i]], self.times, self.names, self.s) for i in r]
+ # for d in x:
+ # for k in ['ims', 'pred', 'xyxy', 'xyxyn', 'xywh', 'xywhn']:
+ # setattr(d, k, getattr(d, k)[0]) # pop out of list
+ return x
+
+ def print(self):
+ LOGGER.info(self.__str__())
+
+ def __len__(self): # override len(results)
+ return self.n
+
+ def __str__(self): # override print(results)
+ return self._run(pprint=True) # print results
+
+ def __repr__(self):
+ return f'YOLO {self.__class__} instance\n' + self.__str__()
+
+
+class Proto(nn.Module):
+ # YOLO mask Proto module for segmentation models
+ def __init__(self, c1, c_=256, c2=32): # ch_in, number of protos, number of masks
+ super().__init__()
+ self.cv1 = Conv(c1, c_, k=3)
+ self.upsample = nn.Upsample(scale_factor=2, mode='nearest')
+ self.cv2 = Conv(c_, c_, k=3)
+ self.cv3 = Conv(c_, c2)
+
+ def forward(self, x):
+ return self.cv3(self.cv2(self.upsample(self.cv1(x))))
+
+
+class UConv(nn.Module):
+ def __init__(self, c1, c_=256, c2=256): # ch_in, number of protos, number of masks
+ super().__init__()
+
+ self.cv1 = Conv(c1, c_, k=3)
+ self.cv2 = nn.Conv2d(c_, c2, 1, 1)
+ self.up = nn.Upsample(scale_factor=2, mode='bilinear', align_corners=True)
+
+ def forward(self, x):
+ return self.up(self.cv2(self.cv1(x)))
+
+
+class Classify(nn.Module):
+ # YOLO classification head, i.e. x(b,c1,20,20) to x(b,c2)
+ def __init__(self, c1, c2, k=1, s=1, p=None, g=1): # ch_in, ch_out, kernel, stride, padding, groups
+ super().__init__()
+ c_ = 1280 # efficientnet_b0 size
+ self.conv = Conv(c1, c_, k, s, autopad(k, p), g)
+ self.pool = nn.AdaptiveAvgPool2d(1) # to x(b,c_,1,1)
+ self.drop = nn.Dropout(p=0.0, inplace=True)
+ self.linear = nn.Linear(c_, c2) # to x(b,c2)
+
+ def forward(self, x):
+ if isinstance(x, list):
+ x = torch.cat(x, 1)
+ return self.linear(self.drop(self.pool(self.conv(x)).flatten(1)))
diff --git a/models/detect/gelan-c.yaml b/models/detect/gelan-c.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..78b41bc39389f633176eccb1f36b685e37ff4347
--- /dev/null
+++ b/models/detect/gelan-c.yaml
@@ -0,0 +1,80 @@
+# YOLOv9
+
+# parameters
+nc: 80 # number of classes
+depth_multiple: 1.0 # model depth multiple
+width_multiple: 1.0 # layer channel multiple
+#activation: nn.LeakyReLU(0.1)
+#activation: nn.ReLU()
+
+# anchors
+anchors: 3
+
+# gelan backbone
+backbone:
+ [
+ # conv down
+ [-1, 1, Conv, [64, 3, 2]], # 0-P1/2
+
+ # conv down
+ [-1, 1, Conv, [128, 3, 2]], # 1-P2/4
+
+ # elan-1 block
+ [-1, 1, RepNCSPELAN4, [256, 128, 64, 1]], # 2
+
+ # avg-conv down
+ [-1, 1, ADown, [256]], # 3-P3/8
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 256, 128, 1]], # 4
+
+ # avg-conv down
+ [-1, 1, ADown, [512]], # 5-P4/16
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 512, 256, 1]], # 6
+
+ # avg-conv down
+ [-1, 1, ADown, [512]], # 7-P5/32
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 512, 256, 1]], # 8
+ ]
+
+# gelan head
+head:
+ [
+ # elan-spp block
+ [-1, 1, SPPELAN, [512, 256]], # 9
+
+ # up-concat merge
+ [-1, 1, nn.Upsample, [None, 2, 'nearest']],
+ [[-1, 6], 1, Concat, [1]], # cat backbone P4
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 512, 256, 1]], # 12
+
+ # up-concat merge
+ [-1, 1, nn.Upsample, [None, 2, 'nearest']],
+ [[-1, 4], 1, Concat, [1]], # cat backbone P3
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [256, 256, 128, 1]], # 15 (P3/8-small)
+
+ # avg-conv-down merge
+ [-1, 1, ADown, [256]],
+ [[-1, 12], 1, Concat, [1]], # cat head P4
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 512, 256, 1]], # 18 (P4/16-medium)
+
+ # avg-conv-down merge
+ [-1, 1, ADown, [512]],
+ [[-1, 9], 1, Concat, [1]], # cat head P5
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 512, 256, 1]], # 21 (P5/32-large)
+
+ # detect
+ [[15, 18, 21], 1, DDetect, [nc]], # DDetect(P3, P4, P5)
+ ]
diff --git a/models/detect/gelan-e.yaml b/models/detect/gelan-e.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..a0409bab70a6c697b0037212b44b0f25de7c1525
--- /dev/null
+++ b/models/detect/gelan-e.yaml
@@ -0,0 +1,121 @@
+# YOLOv9
+
+# parameters
+nc: 80 # number of classes
+depth_multiple: 1.0 # model depth multiple
+width_multiple: 1.0 # layer channel multiple
+#activation: nn.LeakyReLU(0.1)
+#activation: nn.ReLU()
+
+# anchors
+anchors: 3
+
+# gelan backbone
+backbone:
+ [
+ [-1, 1, Silence, []],
+
+ # conv down
+ [-1, 1, Conv, [64, 3, 2]], # 1-P1/2
+
+ # conv down
+ [-1, 1, Conv, [128, 3, 2]], # 2-P2/4
+
+ # elan-1 block
+ [-1, 1, RepNCSPELAN4, [256, 128, 64, 2]], # 3
+
+ # avg-conv down
+ [-1, 1, ADown, [256]], # 4-P3/8
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 256, 128, 2]], # 5
+
+ # avg-conv down
+ [-1, 1, ADown, [512]], # 6-P4/16
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [1024, 512, 256, 2]], # 7
+
+ # avg-conv down
+ [-1, 1, ADown, [1024]], # 8-P5/32
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [1024, 512, 256, 2]], # 9
+
+ # routing
+ [1, 1, CBLinear, [[64]]], # 10
+ [3, 1, CBLinear, [[64, 128]]], # 11
+ [5, 1, CBLinear, [[64, 128, 256]]], # 12
+ [7, 1, CBLinear, [[64, 128, 256, 512]]], # 13
+ [9, 1, CBLinear, [[64, 128, 256, 512, 1024]]], # 14
+
+ # conv down fuse
+ [0, 1, Conv, [64, 3, 2]], # 15-P1/2
+ [[10, 11, 12, 13, 14, -1], 1, CBFuse, [[0, 0, 0, 0, 0]]], # 16
+
+ # conv down fuse
+ [-1, 1, Conv, [128, 3, 2]], # 17-P2/4
+ [[11, 12, 13, 14, -1], 1, CBFuse, [[1, 1, 1, 1]]], # 18
+
+ # elan-1 block
+ [-1, 1, RepNCSPELAN4, [256, 128, 64, 2]], # 19
+
+ # avg-conv down fuse
+ [-1, 1, ADown, [256]], # 20-P3/8
+ [[12, 13, 14, -1], 1, CBFuse, [[2, 2, 2]]], # 21
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 256, 128, 2]], # 22
+
+ # avg-conv down fuse
+ [-1, 1, ADown, [512]], # 23-P4/16
+ [[13, 14, -1], 1, CBFuse, [[3, 3]]], # 24
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [1024, 512, 256, 2]], # 25
+
+ # avg-conv down fuse
+ [-1, 1, ADown, [1024]], # 26-P5/32
+ [[14, -1], 1, CBFuse, [[4]]], # 27
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [1024, 512, 256, 2]], # 28
+ ]
+
+# gelan head
+head:
+ [
+ # elan-spp block
+ [28, 1, SPPELAN, [512, 256]], # 29
+
+ # up-concat merge
+ [-1, 1, nn.Upsample, [None, 2, 'nearest']],
+ [[-1, 25], 1, Concat, [1]], # cat backbone P4
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 512, 256, 2]], # 32
+
+ # up-concat merge
+ [-1, 1, nn.Upsample, [None, 2, 'nearest']],
+ [[-1, 22], 1, Concat, [1]], # cat backbone P3
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [256, 256, 128, 2]], # 35 (P3/8-small)
+
+ # avg-conv-down merge
+ [-1, 1, ADown, [256]],
+ [[-1, 32], 1, Concat, [1]], # cat head P4
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 512, 256, 2]], # 38 (P4/16-medium)
+
+ # avg-conv-down merge
+ [-1, 1, ADown, [512]],
+ [[-1, 29], 1, Concat, [1]], # cat head P5
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 1024, 512, 2]], # 41 (P5/32-large)
+
+ # detect
+ [[35, 38, 41], 1, DDetect, [nc]], # Detect(P3, P4, P5)
+ ]
diff --git a/models/detect/gelan-m.yaml b/models/detect/gelan-m.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..f7a2bc4d0df2f635882db40966120fe659af915f
--- /dev/null
+++ b/models/detect/gelan-m.yaml
@@ -0,0 +1,80 @@
+# YOLOv9
+
+# parameters
+nc: 80 # number of classes
+depth_multiple: 1.0 # model depth multiple
+width_multiple: 1.0 # layer channel multiple
+#activation: nn.LeakyReLU(0.1)
+#activation: nn.ReLU()
+
+# anchors
+anchors: 3
+
+# gelan backbone
+backbone:
+ [
+ # conv down
+ [-1, 1, Conv, [32, 3, 2]], # 0-P1/2
+
+ # conv down
+ [-1, 1, Conv, [64, 3, 2]], # 1-P2/4
+
+ # elan-1 block
+ [-1, 1, RepNCSPELAN4, [128, 128, 64, 1]], # 2
+
+ # avg-conv down
+ [-1, 1, AConv, [240]], # 3-P3/8
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [240, 240, 120, 1]], # 4
+
+ # avg-conv down
+ [-1, 1, AConv, [360]], # 5-P4/16
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [360, 360, 180, 1]], # 6
+
+ # avg-conv down
+ [-1, 1, AConv, [480]], # 7-P5/32
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [480, 480, 240, 1]], # 8
+ ]
+
+# elan head
+head:
+ [
+ # elan-spp block
+ [-1, 1, SPPELAN, [480, 240]], # 9
+
+ # up-concat merge
+ [-1, 1, nn.Upsample, [None, 2, 'nearest']],
+ [[-1, 6], 1, Concat, [1]], # cat backbone P4
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [360, 360, 180, 1]], # 12
+
+ # up-concat merge
+ [-1, 1, nn.Upsample, [None, 2, 'nearest']],
+ [[-1, 4], 1, Concat, [1]], # cat backbone P3
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [240, 240, 120, 1]], # 15
+
+ # avg-conv-down merge
+ [-1, 1, AConv, [180]],
+ [[-1, 12], 1, Concat, [1]], # cat head P4
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [360, 360, 180, 1]], # 18 (P4/16-medium)
+
+ # avg-conv-down merge
+ [-1, 1, AConv, [240]],
+ [[-1, 9], 1, Concat, [1]], # cat head P5
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [480, 480, 240, 1]], # 21 (P5/32-large)
+
+ # detect
+ [[15, 18, 21], 1, DDetect, [nc]], # DDetect(P3, P4, P5)
+ ]
diff --git a/models/detect/gelan-s.yaml b/models/detect/gelan-s.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..0e8d6535aa9aa683f4b3522489821e30ab44d558
--- /dev/null
+++ b/models/detect/gelan-s.yaml
@@ -0,0 +1,80 @@
+# YOLOv9
+
+# parameters
+nc: 80 # number of classes
+depth_multiple: 1.0 # model depth multiple
+width_multiple: 1.0 # layer channel multiple
+#activation: nn.LeakyReLU(0.1)
+#activation: nn.ReLU()
+
+# anchors
+anchors: 3
+
+# gelan backbone
+backbone:
+ [
+ # conv down
+ [-1, 1, Conv, [32, 3, 2]], # 0-P1/2
+
+ # conv down
+ [-1, 1, Conv, [64, 3, 2]], # 1-P2/4
+
+ # elan-1 block
+ [-1, 1, ELAN1, [64, 64, 32]], # 2
+
+ # avg-conv down
+ [-1, 1, AConv, [128]], # 3-P3/8
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [128, 128, 64, 3]], # 4
+
+ # avg-conv down
+ [-1, 1, AConv, [192]], # 5-P4/16
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [192, 192, 96, 3]], # 6
+
+ # avg-conv down
+ [-1, 1, AConv, [256]], # 7-P5/32
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [256, 256, 128, 3]], # 8
+ ]
+
+# elan head
+head:
+ [
+ # elan-spp block
+ [-1, 1, SPPELAN, [256, 128]], # 9
+
+ # up-concat merge
+ [-1, 1, nn.Upsample, [None, 2, 'nearest']],
+ [[-1, 6], 1, Concat, [1]], # cat backbone P4
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [192, 192, 96, 3]], # 12
+
+ # up-concat merge
+ [-1, 1, nn.Upsample, [None, 2, 'nearest']],
+ [[-1, 4], 1, Concat, [1]], # cat backbone P3
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [128, 128, 64, 3]], # 15
+
+ # avg-conv-down merge
+ [-1, 1, AConv, [96]],
+ [[-1, 12], 1, Concat, [1]], # cat head P4
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [192, 192, 96, 3]], # 18 (P4/16-medium)
+
+ # avg-conv-down merge
+ [-1, 1, AConv, [128]],
+ [[-1, 9], 1, Concat, [1]], # cat head P5
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [256, 256, 128, 3]], # 21 (P5/32-large)
+
+ # detect
+ [[15, 18, 21], 1, DDetect, [nc]], # DDetect(P3, P4, P5)
+ ]
diff --git a/models/detect/gelan.yaml b/models/detect/gelan.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..a6f4cad8583c60bbceb3427f47ed046dd2bb24f4
--- /dev/null
+++ b/models/detect/gelan.yaml
@@ -0,0 +1,80 @@
+# YOLOv9
+
+# parameters
+nc: 80 # number of classes
+depth_multiple: 1.0 # model depth multiple
+width_multiple: 1.0 # layer channel multiple
+#activation: nn.LeakyReLU(0.1)
+activation: nn.ReLU()
+
+# anchors
+anchors: 3
+
+# gelan backbone
+backbone:
+ [
+ # conv down
+ [-1, 1, Conv, [64, 3, 2]], # 0-P1/2
+
+ # conv down
+ [-1, 1, Conv, [128, 3, 2]], # 1-P2/4
+
+ # elan-1 block
+ [-1, 1, RepNCSPELAN4, [256, 128, 64, 1]], # 2
+
+ # avg-conv down
+ [-1, 1, Conv, [256, 3, 2]], # 3-P3/8
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 256, 128, 1]], # 4
+
+ # avg-conv down
+ [-1, 1, Conv, [512, 3, 2]], # 5-P4/16
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 512, 256, 1]], # 6
+
+ # avg-conv down
+ [-1, 1, Conv, [512, 3, 2]], # 7-P5/32
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 512, 256, 1]], # 8
+ ]
+
+# gelan head
+head:
+ [
+ # elan-spp block
+ [-1, 1, SPPELAN, [512, 256]], # 9
+
+ # up-concat merge
+ [-1, 1, nn.Upsample, [None, 2, 'nearest']],
+ [[-1, 6], 1, Concat, [1]], # cat backbone P4
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 512, 256, 1]], # 12
+
+ # up-concat merge
+ [-1, 1, nn.Upsample, [None, 2, 'nearest']],
+ [[-1, 4], 1, Concat, [1]], # cat backbone P3
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [256, 256, 128, 1]], # 15 (P3/8-small)
+
+ # avg-conv-down merge
+ [-1, 1, Conv, [256, 3, 2]],
+ [[-1, 12], 1, Concat, [1]], # cat head P4
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 512, 256, 1]], # 18 (P4/16-medium)
+
+ # avg-conv-down merge
+ [-1, 1, Conv, [512, 3, 2]],
+ [[-1, 9], 1, Concat, [1]], # cat head P5
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 512, 256, 1]], # 21 (P5/32-large)
+
+ # detect
+ [[15, 18, 21], 1, DDetect, [nc]], # Detect(P3, P4, P5)
+ ]
diff --git a/models/detect/yolov7-af.yaml b/models/detect/yolov7-af.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..f739df15e6a19d4d4c6b5ed7e912fa6ef3064e59
--- /dev/null
+++ b/models/detect/yolov7-af.yaml
@@ -0,0 +1,137 @@
+# YOLOv7
+
+# Parameters
+nc: 80 # number of classes
+depth_multiple: 1. # model depth multiple
+width_multiple: 1. # layer channel multiple
+anchors: 3
+
+# YOLOv7 backbone
+backbone:
+ # [from, number, module, args]
+ [[-1, 1, Conv, [32, 3, 1]], # 0
+
+ [-1, 1, Conv, [64, 3, 2]], # 1-P1/2
+ [-1, 1, Conv, [64, 3, 1]],
+
+ [-1, 1, Conv, [128, 3, 2]], # 3-P2/4
+ [-1, 1, Conv, [64, 1, 1]],
+ [-2, 1, Conv, [64, 1, 1]],
+ [-1, 1, Conv, [64, 3, 1]],
+ [-1, 1, Conv, [64, 3, 1]],
+ [-1, 1, Conv, [64, 3, 1]],
+ [-1, 1, Conv, [64, 3, 1]],
+ [[-1, -3, -5, -6], 1, Concat, [1]],
+ [-1, 1, Conv, [256, 1, 1]], # 11
+
+ [-1, 1, MP, []],
+ [-1, 1, Conv, [128, 1, 1]],
+ [-3, 1, Conv, [128, 1, 1]],
+ [-1, 1, Conv, [128, 3, 2]],
+ [[-1, -3], 1, Concat, [1]], # 16-P3/8
+ [-1, 1, Conv, [128, 1, 1]],
+ [-2, 1, Conv, [128, 1, 1]],
+ [-1, 1, Conv, [128, 3, 1]],
+ [-1, 1, Conv, [128, 3, 1]],
+ [-1, 1, Conv, [128, 3, 1]],
+ [-1, 1, Conv, [128, 3, 1]],
+ [[-1, -3, -5, -6], 1, Concat, [1]],
+ [-1, 1, Conv, [512, 1, 1]], # 24
+
+ [-1, 1, MP, []],
+ [-1, 1, Conv, [256, 1, 1]],
+ [-3, 1, Conv, [256, 1, 1]],
+ [-1, 1, Conv, [256, 3, 2]],
+ [[-1, -3], 1, Concat, [1]], # 29-P4/16
+ [-1, 1, Conv, [256, 1, 1]],
+ [-2, 1, Conv, [256, 1, 1]],
+ [-1, 1, Conv, [256, 3, 1]],
+ [-1, 1, Conv, [256, 3, 1]],
+ [-1, 1, Conv, [256, 3, 1]],
+ [-1, 1, Conv, [256, 3, 1]],
+ [[-1, -3, -5, -6], 1, Concat, [1]],
+ [-1, 1, Conv, [1024, 1, 1]], # 37
+
+ [-1, 1, MP, []],
+ [-1, 1, Conv, [512, 1, 1]],
+ [-3, 1, Conv, [512, 1, 1]],
+ [-1, 1, Conv, [512, 3, 2]],
+ [[-1, -3], 1, Concat, [1]], # 42-P5/32
+ [-1, 1, Conv, [256, 1, 1]],
+ [-2, 1, Conv, [256, 1, 1]],
+ [-1, 1, Conv, [256, 3, 1]],
+ [-1, 1, Conv, [256, 3, 1]],
+ [-1, 1, Conv, [256, 3, 1]],
+ [-1, 1, Conv, [256, 3, 1]],
+ [[-1, -3, -5, -6], 1, Concat, [1]],
+ [-1, 1, Conv, [1024, 1, 1]], # 50
+ ]
+
+# yolov7 head
+head:
+ [[-1, 1, SPPCSPC, [512]], # 51
+
+ [-1, 1, Conv, [256, 1, 1]],
+ [-1, 1, nn.Upsample, [None, 2, 'nearest']],
+ [37, 1, Conv, [256, 1, 1]], # route backbone P4
+ [[-1, -2], 1, Concat, [1]],
+
+ [-1, 1, Conv, [256, 1, 1]],
+ [-2, 1, Conv, [256, 1, 1]],
+ [-1, 1, Conv, [128, 3, 1]],
+ [-1, 1, Conv, [128, 3, 1]],
+ [-1, 1, Conv, [128, 3, 1]],
+ [-1, 1, Conv, [128, 3, 1]],
+ [[-1, -2, -3, -4, -5, -6], 1, Concat, [1]],
+ [-1, 1, Conv, [256, 1, 1]], # 63
+
+ [-1, 1, Conv, [128, 1, 1]],
+ [-1, 1, nn.Upsample, [None, 2, 'nearest']],
+ [24, 1, Conv, [128, 1, 1]], # route backbone P3
+ [[-1, -2], 1, Concat, [1]],
+
+ [-1, 1, Conv, [128, 1, 1]],
+ [-2, 1, Conv, [128, 1, 1]],
+ [-1, 1, Conv, [64, 3, 1]],
+ [-1, 1, Conv, [64, 3, 1]],
+ [-1, 1, Conv, [64, 3, 1]],
+ [-1, 1, Conv, [64, 3, 1]],
+ [[-1, -2, -3, -4, -5, -6], 1, Concat, [1]],
+ [-1, 1, Conv, [128, 1, 1]], # 75
+
+ [-1, 1, MP, []],
+ [-1, 1, Conv, [128, 1, 1]],
+ [-3, 1, Conv, [128, 1, 1]],
+ [-1, 1, Conv, [128, 3, 2]],
+ [[-1, -3, 63], 1, Concat, [1]],
+
+ [-1, 1, Conv, [256, 1, 1]],
+ [-2, 1, Conv, [256, 1, 1]],
+ [-1, 1, Conv, [128, 3, 1]],
+ [-1, 1, Conv, [128, 3, 1]],
+ [-1, 1, Conv, [128, 3, 1]],
+ [-1, 1, Conv, [128, 3, 1]],
+ [[-1, -2, -3, -4, -5, -6], 1, Concat, [1]],
+ [-1, 1, Conv, [256, 1, 1]], # 88
+
+ [-1, 1, MP, []],
+ [-1, 1, Conv, [256, 1, 1]],
+ [-3, 1, Conv, [256, 1, 1]],
+ [-1, 1, Conv, [256, 3, 2]],
+ [[-1, -3, 51], 1, Concat, [1]],
+
+ [-1, 1, Conv, [512, 1, 1]],
+ [-2, 1, Conv, [512, 1, 1]],
+ [-1, 1, Conv, [256, 3, 1]],
+ [-1, 1, Conv, [256, 3, 1]],
+ [-1, 1, Conv, [256, 3, 1]],
+ [-1, 1, Conv, [256, 3, 1]],
+ [[-1, -2, -3, -4, -5, -6], 1, Concat, [1]],
+ [-1, 1, Conv, [512, 1, 1]], # 101
+
+ [75, 1, Conv, [256, 3, 1]],
+ [88, 1, Conv, [512, 3, 1]],
+ [101, 1, Conv, [1024, 3, 1]],
+
+ [[102, 103, 104], 1, Detect, [nc]], # Detect(P3, P4, P5)
+ ]
diff --git a/models/detect/yolov9-c.yaml b/models/detect/yolov9-c.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..df8d31d2f1d37c97759caea5da4ab6a86d6d1a17
--- /dev/null
+++ b/models/detect/yolov9-c.yaml
@@ -0,0 +1,124 @@
+# YOLOv9
+
+# parameters
+nc: 80 # number of classes
+depth_multiple: 1.0 # model depth multiple
+width_multiple: 1.0 # layer channel multiple
+#activation: nn.LeakyReLU(0.1)
+#activation: nn.ReLU()
+
+# anchors
+anchors: 3
+
+# YOLOv9 backbone
+backbone:
+ [
+ [-1, 1, Silence, []],
+
+ # conv down
+ [-1, 1, Conv, [64, 3, 2]], # 1-P1/2
+
+ # conv down
+ [-1, 1, Conv, [128, 3, 2]], # 2-P2/4
+
+ # elan-1 block
+ [-1, 1, RepNCSPELAN4, [256, 128, 64, 1]], # 3
+
+ # avg-conv down
+ [-1, 1, ADown, [256]], # 4-P3/8
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 256, 128, 1]], # 5
+
+ # avg-conv down
+ [-1, 1, ADown, [512]], # 6-P4/16
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 512, 256, 1]], # 7
+
+ # avg-conv down
+ [-1, 1, ADown, [512]], # 8-P5/32
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 512, 256, 1]], # 9
+ ]
+
+# YOLOv9 head
+head:
+ [
+ # elan-spp block
+ [-1, 1, SPPELAN, [512, 256]], # 10
+
+ # up-concat merge
+ [-1, 1, nn.Upsample, [None, 2, 'nearest']],
+ [[-1, 7], 1, Concat, [1]], # cat backbone P4
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 512, 256, 1]], # 13
+
+ # up-concat merge
+ [-1, 1, nn.Upsample, [None, 2, 'nearest']],
+ [[-1, 5], 1, Concat, [1]], # cat backbone P3
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [256, 256, 128, 1]], # 16 (P3/8-small)
+
+ # avg-conv-down merge
+ [-1, 1, ADown, [256]],
+ [[-1, 13], 1, Concat, [1]], # cat head P4
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 512, 256, 1]], # 19 (P4/16-medium)
+
+ # avg-conv-down merge
+ [-1, 1, ADown, [512]],
+ [[-1, 10], 1, Concat, [1]], # cat head P5
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 512, 256, 1]], # 22 (P5/32-large)
+
+
+ # multi-level reversible auxiliary branch
+
+ # routing
+ [5, 1, CBLinear, [[256]]], # 23
+ [7, 1, CBLinear, [[256, 512]]], # 24
+ [9, 1, CBLinear, [[256, 512, 512]]], # 25
+
+ # conv down
+ [0, 1, Conv, [64, 3, 2]], # 26-P1/2
+
+ # conv down
+ [-1, 1, Conv, [128, 3, 2]], # 27-P2/4
+
+ # elan-1 block
+ [-1, 1, RepNCSPELAN4, [256, 128, 64, 1]], # 28
+
+ # avg-conv down fuse
+ [-1, 1, ADown, [256]], # 29-P3/8
+ [[23, 24, 25, -1], 1, CBFuse, [[0, 0, 0]]], # 30
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 256, 128, 1]], # 31
+
+ # avg-conv down fuse
+ [-1, 1, ADown, [512]], # 32-P4/16
+ [[24, 25, -1], 1, CBFuse, [[1, 1]]], # 33
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 512, 256, 1]], # 34
+
+ # avg-conv down fuse
+ [-1, 1, ADown, [512]], # 35-P5/32
+ [[25, -1], 1, CBFuse, [[2]]], # 36
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 512, 256, 1]], # 37
+
+
+
+ # detection head
+
+ # detect
+ [[31, 34, 37, 16, 19, 22], 1, DualDDetect, [nc]], # DualDDetect(A3, A4, A5, P3, P4, P5)
+ ]
diff --git a/models/detect/yolov9-cf.yaml b/models/detect/yolov9-cf.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..88e0080a98caf20fa8fc78e3de7d26fa8c3ab7f0
--- /dev/null
+++ b/models/detect/yolov9-cf.yaml
@@ -0,0 +1,124 @@
+# YOLOv9
+
+# parameters
+nc: 80 # number of classes
+depth_multiple: 1.0 # model depth multiple
+width_multiple: 1.0 # layer channel multiple
+#activation: nn.LeakyReLU(0.1)
+#activation: nn.ReLU()
+
+# anchors
+anchors: 3
+
+# YOLOv9 backbone
+backbone:
+ [
+ [-1, 1, Silence, []],
+
+ # conv down
+ [-1, 1, Conv, [64, 3, 2]], # 1-P1/2
+
+ # conv down
+ [-1, 1, Conv, [128, 3, 2]], # 2-P2/4
+
+ # elan-1 block
+ [-1, 1, RepNCSPELAN4, [256, 128, 64, 1]], # 3
+
+ # avg-conv down
+ [-1, 1, ADown, [256]], # 4-P3/8
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 256, 128, 1]], # 5
+
+ # avg-conv down
+ [-1, 1, ADown, [512]], # 6-P4/16
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 512, 256, 1]], # 7
+
+ # avg-conv down
+ [-1, 1, ADown, [512]], # 8-P5/32
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 512, 256, 1]], # 9
+ ]
+
+# YOLOv9 head
+head:
+ [
+ # elan-spp block
+ [-1, 1, SPPELAN, [512, 256]], # 10
+
+ # up-concat merge
+ [-1, 1, nn.Upsample, [None, 2, 'nearest']],
+ [[-1, 7], 1, Concat, [1]], # cat backbone P4
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 512, 256, 1]], # 13
+
+ # up-concat merge
+ [-1, 1, nn.Upsample, [None, 2, 'nearest']],
+ [[-1, 5], 1, Concat, [1]], # cat backbone P3
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [256, 256, 128, 1]], # 16 (P3/8-small)
+
+ # avg-conv-down merge
+ [-1, 1, ADown, [256]],
+ [[-1, 13], 1, Concat, [1]], # cat head P4
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 512, 256, 1]], # 19 (P4/16-medium)
+
+ # avg-conv-down merge
+ [-1, 1, ADown, [512]],
+ [[-1, 10], 1, Concat, [1]], # cat head P5
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 512, 256, 1]], # 22 (P5/32-large)
+
+
+ # multi-level reversible auxiliary branch
+
+ # routing
+ [5, 1, CBLinear, [[256]]], # 23
+ [7, 1, CBLinear, [[256, 512]]], # 24
+ [9, 1, CBLinear, [[256, 512, 512]]], # 25
+
+ # conv down
+ [0, 1, Conv, [64, 3, 2]], # 26-P1/2
+
+ # conv down
+ [-1, 1, Conv, [128, 3, 2]], # 27-P2/4
+
+ # elan-1 block
+ [-1, 1, RepNCSPELAN4, [256, 128, 64, 1]], # 28
+
+ # avg-conv down fuse
+ [-1, 1, ADown, [256]], # 29-P3/8
+ [[23, 24, 25, -1], 1, CBFuse, [[0, 0, 0]]], # 30
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 256, 128, 1]], # 31
+
+ # avg-conv down fuse
+ [-1, 1, ADown, [512]], # 32-P4/16
+ [[24, 25, -1], 1, CBFuse, [[1, 1]]], # 33
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 512, 256, 1]], # 34
+
+ # avg-conv down fuse
+ [-1, 1, ADown, [512]], # 35-P5/32
+ [[25, -1], 1, CBFuse, [[2]]], # 36
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 512, 256, 1]], # 37
+
+
+
+ # detection head
+
+ # detect
+ [[31, 34, 37, 16, 19, 22, 16, 19, 22], 1, TripleDDetect, [nc]], # TripleDDetect(A3, A4, A5, P3, P4, P5, P3, P4, P5) Auxiliary/Coarse(NMS-based)/Fine(NMS-free)
+ ]
diff --git a/models/detect/yolov9-e.yaml b/models/detect/yolov9-e.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..dcb122b61cf02eb3f599414f3b7431ddf7b9b898
--- /dev/null
+++ b/models/detect/yolov9-e.yaml
@@ -0,0 +1,144 @@
+# YOLOv9
+
+# parameters
+nc: 80 # number of classes
+depth_multiple: 1.0 # model depth multiple
+width_multiple: 1.0 # layer channel multiple
+#activation: nn.LeakyReLU(0.1)
+#activation: nn.ReLU()
+
+# anchors
+anchors: 3
+
+# YOLOv9 backbone
+backbone:
+ [
+ [-1, 1, Silence, []],
+
+ # conv down
+ [-1, 1, Conv, [64, 3, 2]], # 1-P1/2
+
+ # conv down
+ [-1, 1, Conv, [128, 3, 2]], # 2-P2/4
+
+ # csp-elan block
+ [-1, 1, RepNCSPELAN4, [256, 128, 64, 2]], # 3
+
+ # avg-conv down
+ [-1, 1, ADown, [256]], # 4-P3/8
+
+ # csp-elan block
+ [-1, 1, RepNCSPELAN4, [512, 256, 128, 2]], # 5
+
+ # avg-conv down
+ [-1, 1, ADown, [512]], # 6-P4/16
+
+ # csp-elan block
+ [-1, 1, RepNCSPELAN4, [1024, 512, 256, 2]], # 7
+
+ # avg-conv down
+ [-1, 1, ADown, [1024]], # 8-P5/32
+
+ # csp-elan block
+ [-1, 1, RepNCSPELAN4, [1024, 512, 256, 2]], # 9
+
+ # routing
+ [1, 1, CBLinear, [[64]]], # 10
+ [3, 1, CBLinear, [[64, 128]]], # 11
+ [5, 1, CBLinear, [[64, 128, 256]]], # 12
+ [7, 1, CBLinear, [[64, 128, 256, 512]]], # 13
+ [9, 1, CBLinear, [[64, 128, 256, 512, 1024]]], # 14
+
+ # conv down
+ [0, 1, Conv, [64, 3, 2]], # 15-P1/2
+ [[10, 11, 12, 13, 14, -1], 1, CBFuse, [[0, 0, 0, 0, 0]]], # 16
+
+ # conv down
+ [-1, 1, Conv, [128, 3, 2]], # 17-P2/4
+ [[11, 12, 13, 14, -1], 1, CBFuse, [[1, 1, 1, 1]]], # 18
+
+ # csp-elan block
+ [-1, 1, RepNCSPELAN4, [256, 128, 64, 2]], # 19
+
+ # avg-conv down fuse
+ [-1, 1, ADown, [256]], # 20-P3/8
+ [[12, 13, 14, -1], 1, CBFuse, [[2, 2, 2]]], # 21
+
+ # csp-elan block
+ [-1, 1, RepNCSPELAN4, [512, 256, 128, 2]], # 22
+
+ # avg-conv down fuse
+ [-1, 1, ADown, [512]], # 23-P4/16
+ [[13, 14, -1], 1, CBFuse, [[3, 3]]], # 24
+
+ # csp-elan block
+ [-1, 1, RepNCSPELAN4, [1024, 512, 256, 2]], # 25
+
+ # avg-conv down fuse
+ [-1, 1, ADown, [1024]], # 26-P5/32
+ [[14, -1], 1, CBFuse, [[4]]], # 27
+
+ # csp-elan block
+ [-1, 1, RepNCSPELAN4, [1024, 512, 256, 2]], # 28
+ ]
+
+# YOLOv9 head
+head:
+ [
+ # multi-level auxiliary branch
+
+ # elan-spp block
+ [9, 1, SPPELAN, [512, 256]], # 29
+
+ # up-concat merge
+ [-1, 1, nn.Upsample, [None, 2, 'nearest']],
+ [[-1, 7], 1, Concat, [1]], # cat backbone P4
+
+ # csp-elan block
+ [-1, 1, RepNCSPELAN4, [512, 512, 256, 2]], # 32
+
+ # up-concat merge
+ [-1, 1, nn.Upsample, [None, 2, 'nearest']],
+ [[-1, 5], 1, Concat, [1]], # cat backbone P3
+
+ # csp-elan block
+ [-1, 1, RepNCSPELAN4, [256, 256, 128, 2]], # 35
+
+
+
+ # main branch
+
+ # elan-spp block
+ [28, 1, SPPELAN, [512, 256]], # 36
+
+ # up-concat merge
+ [-1, 1, nn.Upsample, [None, 2, 'nearest']],
+ [[-1, 25], 1, Concat, [1]], # cat backbone P4
+
+ # csp-elan block
+ [-1, 1, RepNCSPELAN4, [512, 512, 256, 2]], # 39
+
+ # up-concat merge
+ [-1, 1, nn.Upsample, [None, 2, 'nearest']],
+ [[-1, 22], 1, Concat, [1]], # cat backbone P3
+
+ # csp-elan block
+ [-1, 1, RepNCSPELAN4, [256, 256, 128, 2]], # 42 (P3/8-small)
+
+ # avg-conv-down merge
+ [-1, 1, ADown, [256]],
+ [[-1, 39], 1, Concat, [1]], # cat head P4
+
+ # csp-elan block
+ [-1, 1, RepNCSPELAN4, [512, 512, 256, 2]], # 45 (P4/16-medium)
+
+ # avg-conv-down merge
+ [-1, 1, ADown, [512]],
+ [[-1, 36], 1, Concat, [1]], # cat head P5
+
+ # csp-elan block
+ [-1, 1, RepNCSPELAN4, [512, 1024, 512, 2]], # 48 (P5/32-large)
+
+ # detect
+ [[35, 32, 29, 42, 45, 48], 1, DualDDetect, [nc]], # DualDDetect(A3, A4, A5, P3, P4, P5)
+ ]
diff --git a/models/detect/yolov9-e_2A.yaml b/models/detect/yolov9-e_2A.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..1d6fed1d207f624e9855dcd1519ceec6835dc32b
--- /dev/null
+++ b/models/detect/yolov9-e_2A.yaml
@@ -0,0 +1,152 @@
+# YOLOv9
+
+# parameters
+nc: 80 # number of classes
+depth_multiple: 1.0 # model depth multiple
+width_multiple: 1.0 # layer channel multiple
+#activation: nn.LeakyReLU(0.1)
+#activation: nn.ReLU()
+
+# anchors
+anchors: 3
+
+# YOLOv9 backbone
+backbone:
+ [
+ [-1, 1, Silence, []],
+
+ # conv down
+ [-1, 1, Conv, [64, 3, 2]], # 1-P1/2
+
+ # conv down
+ [-1, 1, Conv, [128, 3, 2]], # 2-P2/4
+
+ # csp-elan block
+ [-1, 1, RepNCSPELAN4, [256, 128, 64, 1]], # 3
+
+ # avg-conv down
+ [-1, 1, ADown, [256]], # 4-P3/8
+
+ # csp-elan block
+ [-1, 1, RepNCSPELAN4, [512, 256, 128, 1]], # 5
+
+ # avg-conv down
+ [-1, 1, ADown, [512]], # 6-P4/16
+
+ # csp-elan block
+ [-1, 1, RepNCSPELAN4, [1024, 512, 256, 1]], # 7
+
+ # avg-conv down
+ [-1, 1, ADown, [1024]], # 8-P5/32
+
+ # csp-elan block
+ [-1, 1, RepNCSPELAN4, [1024, 512, 256, 1]], # 9
+
+ # routing
+ [1, 1, CBLinear, [[64]]], # 10
+ [3, 1, CBLinear, [[64, 128]]], # 11
+ [5, 1, CBLinear, [[64, 128, 256]]], # 12
+ [7, 1, CBLinear, [[64, 128, 256, 512]]], # 13
+ [9, 1, CBLinear, [[64, 128, 256, 512, 1024]]], # 14
+
+ # conv down
+ [0, 1, Conv, [64, 3, 2]], # 15-P1/2
+ [[10, 11, 12, 13, 14, -1], 1, CBFuse, [[0, 0, 0, 0, 0]]], # 16
+
+ # conv down
+ [-1, 1, Conv, [128, 3, 2]], # 17-P2/4
+ [[11, 12, 13, 14, -1], 1, CBFuse, [[1, 1, 1, 1]]], # 18
+
+ # csp-elan block
+ [-1, 1, RepNCSPELAN4, [256, 128, 64, 1]], # 19
+
+ # avg-conv down fuse
+ [-1, 1, ADown, [256]], # 20-P3/8
+ [[12, 13, 14, -1], 1, CBFuse, [[2, 2, 2]]], # 21
+
+ # csp-elan block
+ [-1, 1, RepNCSPELAN4, [512, 256, 128, 1]], # 22
+
+ # avg-conv down fuse
+ [-1, 1, ADown, [512]], # 23-P4/16
+ [[13, 14, -1], 1, CBFuse, [[3, 3]]], # 24
+
+ # csp-elan block
+ [-1, 1, RepNCSPELAN4, [1024, 512, 256, 1]], # 25
+
+ # avg-conv down fuse
+ [-1, 1, ADown, [1024]], # 26-P5/32
+ [[14, -1], 1, CBFuse, [[4]]], # 27
+
+ # csp-elan block
+ [-1, 1, RepNCSPELAN4, [1024, 512, 256, 1]], # 28
+ ]
+
+# YOLOv9 head
+head:
+ [
+ # multi-level auxiliary branch
+
+ # elan-spp block
+ [9, 1, SPPELAN, [512, 256]], # 29
+
+ # up-concat merge
+ [-1, 1, nn.Upsample, [None, 2, 'nearest']],
+ [[-1, 7], 1, Concat, [1]], # cat backbone P4
+
+ # csp-elan block
+ [-1, 1, RepNCSPELAN4, [512, 512, 256, 1]], # 32
+
+ # up-concat merge
+ [-1, 1, nn.Upsample, [None, 2, 'nearest']],
+ [[-1, 5], 1, Concat, [1]], # cat backbone P3
+
+ # csp-elan block
+ [-1, 1, RepNCSPELAN4, [256, 256, 128, 1]], # 35
+
+
+
+ # main branch
+
+ # elan-spp block
+ [28, 1, SPPELAN, [512, 256]], # 36
+
+ # up-concat merge
+ [-1, 1, nn.Upsample, [None, 2, 'nearest']],
+ [[-1, 25], 1, Concat, [1]], # cat backbone P4
+
+ # csp-elan block
+ [-1, 1, RepNCSPELAN4, [512, 512, 256, 1]], # 39
+
+ # up-concat merge
+ [-1, 1, nn.Upsample, [None, 2, 'nearest']],
+ [[-1, 22], 1, Concat, [1]], # cat backbone P3
+
+ # csp-elan block
+ [-1, 1, RepNCSPELAN4, [256, 256, 128, 1]], # 42 (P3/8-small)
+
+ # avg-conv-down merge
+ [-1, 1, ADown, [256]],
+ [[-1, 39], 1, Concat, [1]], # cat head P4
+
+ # csp-elan block
+ [-1, 1, RepNCSPELAN4, [512, 512, 256, 1]], # 45 (P4/16-medium)
+
+ # avg-conv-down merge
+ [-1, 1, ADown, [512]],
+ [[-1, 36], 1, Concat, [1]], # cat head P5
+
+ # csp-elan block
+ [-1, 1, RepNCSPELAN4, [1024, 1024, 512, 1]], # 48 (P5/32-large)
+
+ #transformer block
+ [-1, 1, PSA, [1024,1024]], # 49 trasformerblock
+ [45, 1, PSA, [512,512]], # 50 trasformerblock
+
+
+
+
+
+ # detect
+ [[35, 32, 29, 42, 50, 49], 1, DualDDetect, [nc]], # DualDDetect(A3, A4, A5, P3, P4, P5)
+ ]
diff --git a/models/detect/yolov9-m.yaml b/models/detect/yolov9-m.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..adcf0595715301f0c784a34bc13fb96d0771cde4
--- /dev/null
+++ b/models/detect/yolov9-m.yaml
@@ -0,0 +1,117 @@
+# YOLOv9
+
+# parameters
+nc: 80 # number of classes
+depth_multiple: 1.0 # model depth multiple
+width_multiple: 1.0 # layer channel multiple
+#activation: nn.LeakyReLU(0.1)
+#activation: nn.ReLU()
+
+# anchors
+anchors: 3
+
+# gelan backbone
+backbone:
+ [
+ [-1, 1, Silence, []],
+
+ # conv down
+ [-1, 1, Conv, [32, 3, 2]], # 1-P1/2
+
+ # conv down
+ [-1, 1, Conv, [64, 3, 2]], # 2-P2/4
+
+ # elan-1 block
+ [-1, 1, RepNCSPELAN4, [128, 128, 64, 1]], # 3
+
+ # avg-conv down
+ [-1, 1, AConv, [240]], # 4-P3/8
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [240, 240, 120, 1]], # 5
+
+ # avg-conv down
+ [-1, 1, AConv, [360]], # 6-P4/16
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [360, 360, 180, 1]], # 7
+
+ # avg-conv down
+ [-1, 1, AConv, [480]], # 8-P5/32
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [480, 480, 240, 1]], # 9
+ ]
+
+# elan head
+head:
+ [
+ # elan-spp block
+ [-1, 1, SPPELAN, [480, 240]], # 10
+
+ # up-concat merge
+ [-1, 1, nn.Upsample, [None, 2, 'nearest']],
+ [[-1, 7], 1, Concat, [1]], # cat backbone P4
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [360, 360, 180, 1]], # 13
+
+ # up-concat merge
+ [-1, 1, nn.Upsample, [None, 2, 'nearest']],
+ [[-1, 5], 1, Concat, [1]], # cat backbone P3
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [240, 240, 120, 1]], # 16
+
+ # avg-conv-down merge
+ [-1, 1, AConv, [180]],
+ [[-1, 13], 1, Concat, [1]], # cat head P4
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [360, 360, 180, 1]], # 19 (P4/16-medium)
+
+ # avg-conv-down merge
+ [-1, 1, AConv, [240]],
+ [[-1, 10], 1, Concat, [1]], # cat head P5
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [480, 480, 240, 1]], # 22 (P5/32-large)
+
+ # routing
+ [5, 1, CBLinear, [[240]]], # 23
+ [7, 1, CBLinear, [[240, 360]]], # 24
+ [9, 1, CBLinear, [[240, 360, 480]]], # 25
+
+ # conv down
+ [0, 1, Conv, [32, 3, 2]], # 26-P1/2
+
+ # conv down
+ [-1, 1, Conv, [64, 3, 2]], # 27-P2/4
+
+ # elan-1 block
+ [-1, 1, RepNCSPELAN4, [128, 128, 64, 1]], # 28
+
+ # avg-conv down
+ [-1, 1, AConv, [240]], # 29-P3/8
+ [[23, 24, 25, -1], 1, CBFuse, [[0, 0, 0]]], # 30
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [240, 240, 120, 1]], # 31
+
+ # avg-conv down
+ [-1, 1, AConv, [360]], # 32-P4/16
+ [[24, 25, -1], 1, CBFuse, [[1, 1]]], # 33
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [360, 360, 180, 1]], # 34
+
+ # avg-conv down
+ [-1, 1, AConv, [480]], # 35-P5/32
+ [[25, -1], 1, CBFuse, [[2]]], # 36
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [480, 480, 240, 1]], # 37
+
+ # detect
+ [[31, 34, 37, 16, 19, 22], 1, DualDDetect, [nc]], # Detect(P3, P4, P5)
+ ]
diff --git a/models/detect/yolov9-s.yaml b/models/detect/yolov9-s.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..12c5ee0a73a0cce9a784ca6bebe9c6f3539b1844
--- /dev/null
+++ b/models/detect/yolov9-s.yaml
@@ -0,0 +1,97 @@
+# YOLOv9
+
+# parameters
+nc: 80 # number of classes
+depth_multiple: 1.0 # model depth multiple
+width_multiple: 1.0 # layer channel multiple
+#activation: nn.LeakyReLU(0.1)
+#activation: nn.ReLU()
+
+# anchors
+anchors: 3
+
+# gelan backbone
+backbone:
+ [
+ # conv down
+ [-1, 1, Conv, [32, 3, 2]], # 0-P1/2
+
+ # conv down
+ [-1, 1, Conv, [64, 3, 2]], # 1-P2/4
+
+ # elan-1 block
+ [-1, 1, ELAN1, [64, 64, 32]], # 2
+
+ # avg-conv down
+ [-1, 1, AConv, [128]], # 3-P3/8
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [128, 128, 64, 3]], # 4
+
+ # avg-conv down
+ [-1, 1, AConv, [192]], # 5-P4/16
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [192, 192, 96, 3]], # 6
+
+ # avg-conv down
+ [-1, 1, AConv, [256]], # 7-P5/32
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [256, 256, 128, 3]], # 8
+ ]
+
+# elan head
+head:
+ [
+ # elan-spp block
+ [-1, 1, SPPELAN, [256, 128]], # 9
+
+ # up-concat merge
+ [-1, 1, nn.Upsample, [None, 2, 'nearest']],
+ [[-1, 6], 1, Concat, [1]], # cat backbone P4
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [192, 192, 96, 3]], # 12
+
+ # up-concat merge
+ [-1, 1, nn.Upsample, [None, 2, 'nearest']],
+ [[-1, 4], 1, Concat, [1]], # cat backbone P3
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [128, 128, 64, 3]], # 15
+
+ # avg-conv-down merge
+ [-1, 1, AConv, [96]],
+ [[-1, 12], 1, Concat, [1]], # cat head P4
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [192, 192, 96, 3]], # 18 (P4/16-medium)
+
+ # avg-conv-down merge
+ [-1, 1, AConv, [128]],
+ [[-1, 9], 1, Concat, [1]], # cat head P5
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [256, 256, 128, 3]], # 21 (P5/32-large)
+
+ # elan-spp block
+ [8, 1, SPPELAN, [256, 128]], # 22
+
+ # up-concat merge
+ [-1, 1, nn.Upsample, [None, 2, 'nearest']],
+ [[-1, 6], 1, Concat, [1]], # cat backbone P4
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [192, 192, 96, 3]], # 25
+
+ # up-concat merge
+ [-1, 1, nn.Upsample, [None, 2, 'nearest']],
+ [[-1, 4], 1, Concat, [1]], # cat backbone P3
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [128, 128, 64, 3]], # 28
+
+ # detect
+ [[28, 25, 22, 15, 18, 21], 1, DualDDetect, [nc]], # Detect(P3, P4, P5)
+ ]
diff --git a/models/detect/yolov9.yaml b/models/detect/yolov9.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..98ecd14f64d4758a0ac02f2a6b2fab6a18869bdb
--- /dev/null
+++ b/models/detect/yolov9.yaml
@@ -0,0 +1,117 @@
+# YOLOv9
+
+# parameters
+nc: 80 # number of classes
+depth_multiple: 1.0 # model depth multiple
+width_multiple: 1.0 # layer channel multiple
+#activation: nn.LeakyReLU(0.1)
+activation: nn.ReLU()
+
+# anchors
+anchors: 3
+
+# YOLOv9 backbone
+backbone:
+ [
+ [-1, 1, Silence, []],
+
+ # conv down
+ [-1, 1, Conv, [64, 3, 2]], # 1-P1/2
+
+ # conv down
+ [-1, 1, Conv, [128, 3, 2]], # 2-P2/4
+
+ # elan-1 block
+ [-1, 1, RepNCSPELAN4, [256, 128, 64, 1]], # 3
+
+ # conv down
+ [-1, 1, Conv, [256, 3, 2]], # 4-P3/8
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 256, 128, 1]], # 5
+
+ # conv down
+ [-1, 1, Conv, [512, 3, 2]], # 6-P4/16
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 512, 256, 1]], # 7
+
+ # conv down
+ [-1, 1, Conv, [512, 3, 2]], # 8-P5/32
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 512, 256, 1]], # 9
+ ]
+
+# YOLOv9 head
+head:
+ [
+ # elan-spp block
+ [-1, 1, SPPELAN, [512, 256]], # 10
+
+ # up-concat merge
+ [-1, 1, nn.Upsample, [None, 2, 'nearest']],
+ [[-1, 7], 1, Concat, [1]], # cat backbone P4
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 512, 256, 1]], # 13
+
+ # up-concat merge
+ [-1, 1, nn.Upsample, [None, 2, 'nearest']],
+ [[-1, 5], 1, Concat, [1]], # cat backbone P3
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [256, 256, 128, 1]], # 16 (P3/8-small)
+
+ # conv-down merge
+ [-1, 1, Conv, [256, 3, 2]],
+ [[-1, 13], 1, Concat, [1]], # cat head P4
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 512, 256, 1]], # 19 (P4/16-medium)
+
+ # conv-down merge
+ [-1, 1, Conv, [512, 3, 2]],
+ [[-1, 10], 1, Concat, [1]], # cat head P5
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 512, 256, 1]], # 22 (P5/32-large)
+
+ # routing
+ [5, 1, CBLinear, [[256]]], # 23
+ [7, 1, CBLinear, [[256, 512]]], # 24
+ [9, 1, CBLinear, [[256, 512, 512]]], # 25
+
+ # conv down
+ [0, 1, Conv, [64, 3, 2]], # 26-P1/2
+
+ # conv down
+ [-1, 1, Conv, [128, 3, 2]], # 27-P2/4
+
+ # elan-1 block
+ [-1, 1, RepNCSPELAN4, [256, 128, 64, 1]], # 28
+
+ # conv down fuse
+ [-1, 1, Conv, [256, 3, 2]], # 29-P3/8
+ [[23, 24, 25, -1], 1, CBFuse, [[0, 0, 0]]], # 30
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 256, 128, 1]], # 31
+
+ # conv down fuse
+ [-1, 1, Conv, [512, 3, 2]], # 32-P4/16
+ [[24, 25, -1], 1, CBFuse, [[1, 1]]], # 33
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 512, 256, 1]], # 34
+
+ # conv down fuse
+ [-1, 1, Conv, [512, 3, 2]], # 35-P5/32
+ [[25, -1], 1, CBFuse, [[2]]], # 36
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 512, 256, 1]], # 37
+
+ # detect
+ [[31, 34, 37, 16, 19, 22], 1, DualDDetect, [nc]], # DualDDetect(A3, A4, A5, P3, P4, P5)
+ ]
diff --git a/models/detect/yolov9e-1A.yaml b/models/detect/yolov9e-1A.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..ec28a16b56f2fe7739e13d9deed0228ef2a342fd
--- /dev/null
+++ b/models/detect/yolov9e-1A.yaml
@@ -0,0 +1,146 @@
+# YOLOv9
+
+# parameters
+nc: 80 # number of classes
+depth_multiple: 1.0 # model depth multiple
+width_multiple: 1.0 # layer channel multiple
+#activation: nn.LeakyReLU(0.1)
+#activation: nn.ReLU()
+
+# anchors
+anchors: 3
+
+# YOLOv9 backbone
+backbone:
+ [
+ [-1, 1, Silence, []],
+
+ # conv down
+ [-1, 1, Conv, [64, 3, 2]], # 1-P1/2
+
+ # conv down
+ [-1, 1, Conv, [128, 3, 2]], # 2-P2/4
+
+ # csp-elan block
+ [-1, 1, RepNCSPELAN4, [256, 128, 64, 2]], # 3
+
+ # avg-conv down
+ [-1, 1, ADown, [256]], # 4-P3/8
+
+ # csp-elan block
+ [-1, 1, RepNCSPELAN4, [512, 256, 128, 2]], # 5
+
+ # avg-conv down
+ [-1, 1, ADown, [512]], # 6-P4/16
+
+ # csp-elan block
+ [-1, 1, RepNCSPELAN4, [1024, 512, 256, 2]], # 7
+
+ # avg-conv down
+ [-1, 1, ADown, [1024]], # 8-P5/32
+
+ # csp-elan block
+ [-1, 1, RepNCSPELAN4, [1024, 512, 256, 2]], # 9
+
+ # routing
+ [1, 1, CBLinear, [[64]]], # 10
+ [3, 1, CBLinear, [[64, 128]]], # 11
+ [5, 1, CBLinear, [[64, 128, 256]]], # 12
+ [7, 1, CBLinear, [[64, 128, 256, 512]]], # 13
+ [9, 1, CBLinear, [[64, 128, 256, 512, 1024]]], # 14
+
+ # conv down
+ [0, 1, Conv, [64, 3, 2]], # 15-P1/2
+ [[10, 11, 12, 13, 14, -1], 1, CBFuse, [[0, 0, 0, 0, 0]]], # 16
+
+ # conv down
+ [-1, 1, Conv, [128, 3, 2]], # 17-P2/4
+ [[11, 12, 13, 14, -1], 1, CBFuse, [[1, 1, 1, 1]]], # 18
+
+ # csp-elan block
+ [-1, 1, RepNCSPELAN4, [256, 128, 64, 2]], # 19
+
+ # avg-conv down fuse
+ [-1, 1, ADown, [256]], # 20-P3/8
+ [[12, 13, 14, -1], 1, CBFuse, [[2, 2, 2]]], # 21
+
+ # csp-elan block
+ [-1, 1, RepNCSPELAN4, [512, 256, 128, 2]], # 22
+
+ # avg-conv down fuse
+ [-1, 1, ADown, [512]], # 23-P4/16
+ [[13, 14, -1], 1, CBFuse, [[3, 3]]], # 24
+
+ # csp-elan block
+ [-1, 1, RepNCSPELAN4, [1024, 512, 256, 2]], # 25
+
+ # avg-conv down fuse
+ [-1, 1, ADown, [1024]], # 26-P5/32
+ [[14, -1], 1, CBFuse, [[4]]], # 27
+
+ # csp-elan block
+ [-1, 1, RepNCSPELAN4, [1024, 512, 256, 2]], # 28
+ ]
+
+# YOLOv9 head
+head:
+ [
+ # multi-level auxiliary branch
+
+ # elan-spp block
+ [9, 1, SPPELAN, [512, 256]], # 29
+
+ # up-concat merge
+ [-1, 1, nn.Upsample, [None, 2, 'nearest']],
+ [[-1, 7], 1, Concat, [1]], # cat backbone P4
+
+ # csp-elan block
+ [-1, 1, RepNCSPELAN4, [512, 512, 256, 2]], # 32
+
+ # up-concat merge
+ [-1, 1, nn.Upsample, [None, 2, 'nearest']],
+ [[-1, 5], 1, Concat, [1]], # cat backbone P3
+
+ # csp-elan block
+ [-1, 1, RepNCSPELAN4, [256, 256, 128, 2]], # 35
+
+
+
+ # main branch
+
+ # elan-spp block
+ [28, 1, SPPELAN, [512, 256]], # 36
+
+ # up-concat merge
+ [-1, 1, nn.Upsample, [None, 2, 'nearest']],
+ [[-1, 25], 1, Concat, [1]], # cat backbone P4
+
+ # csp-elan block
+ [-1, 1, RepNCSPELAN4, [512, 512, 256, 2]], # 39
+
+ # up-concat merge
+ [-1, 1, nn.Upsample, [None, 2, 'nearest']],
+ [[-1, 22], 1, Concat, [1]], # cat backbone P3
+
+ # csp-elan block
+ [-1, 1, RepNCSPELAN4, [256, 256, 128, 2]], # 42 (P3/8-small)
+
+ # avg-conv-down merge
+ [-1, 1, ADown, [256]],
+ [[-1, 39], 1, Concat, [1]], # cat head P4
+
+ # csp-elan block
+ [-1, 1, RepNCSPELAN4, [512, 512, 256, 2]], # 45 (P4/16-medium)
+
+ # avg-conv-down merge
+ [-1, 1, ADown, [512]],
+ [[-1, 36], 1, Concat, [1]], # cat head P5
+
+ # csp-elan block
+ [-1, 1, RepNCSPELAN4, [1024, 1024, 512, 2]], # 48 (P5/32-large)
+ #transformer block
+ [-1, 1, PSA, [1024,1024]], # 49 trasformerblock
+
+ # detect
+ [[35, 32, 29, 42, 45, 49], 1, DualDDetect, [nc]], # DualDDetect(A3, A4, A5, P3, P4, P5)
+ ]
diff --git a/models/detect/yolov9tr-l.yaml b/models/detect/yolov9tr-l.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..7a53f753d5f823416bafe36dcbbf6366389b2dd6
--- /dev/null
+++ b/models/detect/yolov9tr-l.yaml
@@ -0,0 +1,111 @@
+# YOLOv9
+
+# parameters
+nc: 80 # number of classes
+depth_multiple: 1.0 # model depth multiple
+width_multiple: 1.0 # layer channel multiple
+#activation: nn.LeakyReLU(0.1)
+#activation: nn.ReLU()
+
+# anchors
+anchors: 3
+
+# gelan backbone
+backbone:
+ [
+ # conv down
+ [-1, 1, Conv, [32, 3, 2]], # 0-P1/2
+
+ # conv down
+ [-1, 1, Conv, [64, 3, 2]], # 1-P2/4
+
+ # elan-1 block
+ [-1, 1, ELAN1, [64, 64, 32]], # 2
+
+ # avg-conv down
+ [-1, 1, AConv, [128]], # 3-P3/8
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [128, 128, 64, 3]], # 4
+
+ # avg-conv down
+ [-1, 1, AConv, [192]], # 5-P4/16
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [192, 192, 96, 3]], # 6
+
+ # avg-conv down
+ [-1, 1, AConv, [256]], # 7-P5/32
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [256, 256, 128, 3]], # 8
+ ]
+
+# elan head
+head:
+ [
+ # elan-spp block
+ [-1, 1, SPPELAN, [256, 128]], # 9
+
+ # up-concat merge
+ [-1, 1, nn.Upsample, [None, 2, 'nearest']],
+ [[-1, 6], 1, Concat, [1]], # cat backbone P4
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [192, 192, 96, 3]], # 12
+
+ # up-concat merge
+ [-1, 1, nn.Upsample, [None, 2, 'nearest']],
+ [[-1, 4], 1, Concat, [1]], # cat backbone P3
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [128, 128, 64, 3]], # 15
+
+ # avg-conv-down merge
+ [-1, 1, AConv, [96]],
+ [[-1, 12], 1, Concat, [1]], # cat head P4
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [192, 192, 96, 3]], # 18 (P4/16-medium)
+
+ # avg-conv-down merge
+ [-1, 1, AConv, [128]],
+ [[-1, 9], 1, Concat, [1]], # cat head P5
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [256, 256, 128, 3]], # 21 (P5/32-large)
+
+ # elan-spp block
+ [8, 1, SPPELAN, [256, 128]], # 22
+
+ # up-concat merge
+ [-1, 1, nn.Upsample, [None, 2, 'nearest']],
+ [[-1, 6], 1, Concat, [1]], # cat backbone P4
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [192, 192, 96, 3]], # 25
+
+ # up-concat merge
+ [-1, 1, nn.Upsample, [None, 2, 'nearest']],
+ [[-1, 4], 1, Concat, [1]], # cat backbone P3
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [128, 128, 64, 3]], # 28
+
+ #transformer block
+ [28, 1, PSA, [128,128]], # 29 trasformerblock
+ [25, 1, PSA, [192,192]], # 30 trasformerblock
+ [22, 1, PSA, [256,256]], # 31 trasformerblock
+
+ [15, 1, PSA, [128,128]], # 32 trasformerblock
+ [18, 1, PSA, [192,192]], # 33 trasformerblock
+ [21, 1, PSA, [256,256]], # 34 trasformerblock
+
+
+
+
+
+
+ # detect
+ [[29, 30, 31, 32, 33, 34], 1, DualDDetect, [nc]], # Detect(P3, P4, P5)
+ ]
diff --git a/models/detect/yolov9tr.pt b/models/detect/yolov9tr.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d7f515c51d755ff69c9b58eae9c37e7a4731c415
--- /dev/null
+++ b/models/detect/yolov9tr.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f8937cc567fcb04e06de141c866a54deb05a3fcf6e0747c2815544efab5af86b
+size 83640778
diff --git a/models/detect/yolov9tr.yaml b/models/detect/yolov9tr.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..a36ecba968fc57d69b06d58e3bec3a3081942a48
--- /dev/null
+++ b/models/detect/yolov9tr.yaml
@@ -0,0 +1,105 @@
+# YOLOv9
+
+# parameters
+nc: 80 # number of classes
+depth_multiple: 1.0 # model depth multiple
+width_multiple: 1.0 # layer channel multiple
+#activation: nn.LeakyReLU(0.1)
+#activation: nn.ReLU()
+
+# anchors
+anchors: 3
+
+# gelan backbone
+backbone:
+ [
+ # conv down
+ [-1, 1, Conv, [32, 3, 2]], # 0-P1/2
+
+ # conv down
+ [-1, 1, Conv, [64, 3, 2]], # 1-P2/4
+
+ # elan-1 block
+ [-1, 1, ELAN1, [64, 64, 32]], # 2
+
+ # avg-conv down
+ [-1, 1, AConv, [128]], # 3-P3/8
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [128, 128, 64, 3]], # 4
+
+ # avg-conv down
+ [-1, 1, AConv, [192]], # 5-P4/16
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [192, 192, 96, 3]], # 6
+
+ # avg-conv down
+ [-1, 1, AConv, [256]], # 7-P5/32
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [256, 256, 128, 3]], # 8
+ ]
+
+# elan head
+head:
+ [
+ # elan-spp block
+ [-1, 1, SPPELAN, [256, 128]], # 9
+
+ # up-concat merge
+ [-1, 1, nn.Upsample, [None, 2, 'nearest']],
+ [[-1, 6], 1, Concat, [1]], # cat backbone P4
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [192, 192, 96, 3]], # 12
+
+ # up-concat merge
+ [-1, 1, nn.Upsample, [None, 2, 'nearest']],
+ [[-1, 4], 1, Concat, [1]], # cat backbone P3
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [128, 128, 64, 3]], # 15
+
+ # avg-conv-down merge
+ [-1, 1, AConv, [96]],
+ [[-1, 12], 1, Concat, [1]], # cat head P4
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [192, 192, 96, 3]], # 18 (P4/16-medium)
+
+ # avg-conv-down merge
+ [-1, 1, AConv, [128]],
+ [[-1, 9], 1, Concat, [1]], # cat head P5
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [256, 256, 128, 3]], # 21 (P5/32-large)
+
+ # elan-spp block
+ [8, 1, SPPELAN, [256, 128]], # 22
+
+ # up-concat merge
+ [-1, 1, nn.Upsample, [None, 2, 'nearest']],
+ [[-1, 6], 1, Concat, [1]], # cat backbone P4
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [192, 192, 96, 3]], # 25
+
+ # up-concat merge
+ [-1, 1, nn.Upsample, [None, 2, 'nearest']],
+ [[-1, 4], 1, Concat, [1]], # cat backbone P3
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [128, 128, 64, 3]], # 28
+
+ #transformer block
+ [28, 1, PSA, [128,128]], # 29 trasformerblock
+ [25, 1, PSA, [192,192]], # 30 trasformerblock
+ [22, 1, PSA, [256,256]], # 31 trasformerblock
+
+
+
+
+ # detect
+ [[29, 30, 31, 15, 18, 21], 1, DualDDetect, [nc]], # Detect(P3, P4, P5)
+ ]
diff --git a/models/experimental.py b/models/experimental.py
new file mode 100644
index 0000000000000000000000000000000000000000..b1a466a6ce67cd7751a8c2799a708539f98dfc28
--- /dev/null
+++ b/models/experimental.py
@@ -0,0 +1,275 @@
+import math
+
+import numpy as np
+import torch
+import torch.nn as nn
+
+from utils.downloads import attempt_download
+
+
+class Sum(nn.Module):
+ # Weighted sum of 2 or more layers https://arxiv.org/abs/1911.09070
+ def __init__(self, n, weight=False): # n: number of inputs
+ super().__init__()
+ self.weight = weight # apply weights boolean
+ self.iter = range(n - 1) # iter object
+ if weight:
+ self.w = nn.Parameter(-torch.arange(1.0, n) / 2, requires_grad=True) # layer weights
+
+ def forward(self, x):
+ y = x[0] # no weight
+ if self.weight:
+ w = torch.sigmoid(self.w) * 2
+ for i in self.iter:
+ y = y + x[i + 1] * w[i]
+ else:
+ for i in self.iter:
+ y = y + x[i + 1]
+ return y
+
+
+class MixConv2d(nn.Module):
+ # Mixed Depth-wise Conv https://arxiv.org/abs/1907.09595
+ def __init__(self, c1, c2, k=(1, 3), s=1, equal_ch=True): # ch_in, ch_out, kernel, stride, ch_strategy
+ super().__init__()
+ n = len(k) # number of convolutions
+ if equal_ch: # equal c_ per group
+ i = torch.linspace(0, n - 1E-6, c2).floor() # c2 indices
+ c_ = [(i == g).sum() for g in range(n)] # intermediate channels
+ else: # equal weight.numel() per group
+ b = [c2] + [0] * n
+ a = np.eye(n + 1, n, k=-1)
+ a -= np.roll(a, 1, axis=1)
+ a *= np.array(k) ** 2
+ a[0] = 1
+ c_ = np.linalg.lstsq(a, b, rcond=None)[0].round() # solve for equal weight indices, ax = b
+
+ self.m = nn.ModuleList([
+ nn.Conv2d(c1, int(c_), k, s, k // 2, groups=math.gcd(c1, int(c_)), bias=False) for k, c_ in zip(k, c_)])
+ self.bn = nn.BatchNorm2d(c2)
+ self.act = nn.SiLU()
+
+ def forward(self, x):
+ return self.act(self.bn(torch.cat([m(x) for m in self.m], 1)))
+
+
+class Ensemble(nn.ModuleList):
+ # Ensemble of models
+ def __init__(self):
+ super().__init__()
+
+ def forward(self, x, augment=False, profile=False, visualize=False):
+ y = [module(x, augment, profile, visualize)[0] for module in self]
+ # y = torch.stack(y).max(0)[0] # max ensemble
+ # y = torch.stack(y).mean(0) # mean ensemble
+ y = torch.cat(y, 1) # nms ensemble
+ return y, None # inference, train output
+
+
+class ORT_NMS(torch.autograd.Function):
+ '''ONNX-Runtime NMS operation'''
+ @staticmethod
+ def forward(ctx,
+ boxes,
+ scores,
+ max_output_boxes_per_class=torch.tensor([100]),
+ iou_threshold=torch.tensor([0.45]),
+ score_threshold=torch.tensor([0.25])):
+ device = boxes.device
+ batch = scores.shape[0]
+ num_det = random.randint(0, 100)
+ batches = torch.randint(0, batch, (num_det,)).sort()[0].to(device)
+ idxs = torch.arange(100, 100 + num_det).to(device)
+ zeros = torch.zeros((num_det,), dtype=torch.int64).to(device)
+ selected_indices = torch.cat([batches[None], zeros[None], idxs[None]], 0).T.contiguous()
+ selected_indices = selected_indices.to(torch.int64)
+ return selected_indices
+
+ @staticmethod
+ def symbolic(g, boxes, scores, max_output_boxes_per_class, iou_threshold, score_threshold):
+ return g.op("NonMaxSuppression", boxes, scores, max_output_boxes_per_class, iou_threshold, score_threshold)
+
+
+class TRT_NMS(torch.autograd.Function):
+ '''TensorRT NMS operation'''
+ @staticmethod
+ def forward(
+ ctx,
+ boxes,
+ scores,
+ background_class=-1,
+ box_coding=1,
+ iou_threshold=0.45,
+ max_output_boxes=100,
+ plugin_version="1",
+ score_activation=0,
+ score_threshold=0.25,
+ ):
+
+ batch_size, num_boxes, num_classes = scores.shape
+ num_det = torch.randint(0, max_output_boxes, (batch_size, 1), dtype=torch.int32)
+ det_boxes = torch.randn(batch_size, max_output_boxes, 4)
+ det_scores = torch.randn(batch_size, max_output_boxes)
+ det_classes = torch.randint(0, num_classes, (batch_size, max_output_boxes), dtype=torch.int32)
+ return num_det, det_boxes, det_scores, det_classes
+
+ @staticmethod
+ def symbolic(g,
+ boxes,
+ scores,
+ background_class=-1,
+ box_coding=1,
+ iou_threshold=0.45,
+ max_output_boxes=100,
+ plugin_version="1",
+ score_activation=0,
+ score_threshold=0.25):
+ out = g.op("TRT::EfficientNMS_TRT",
+ boxes,
+ scores,
+ background_class_i=background_class,
+ box_coding_i=box_coding,
+ iou_threshold_f=iou_threshold,
+ max_output_boxes_i=max_output_boxes,
+ plugin_version_s=plugin_version,
+ score_activation_i=score_activation,
+ score_threshold_f=score_threshold,
+ outputs=4)
+ nums, boxes, scores, classes = out
+ return nums, boxes, scores, classes
+
+
+class ONNX_ORT(nn.Module):
+ '''onnx module with ONNX-Runtime NMS operation.'''
+ def __init__(self, max_obj=100, iou_thres=0.45, score_thres=0.25, max_wh=640, device=None, n_classes=80):
+ super().__init__()
+ self.device = device if device else torch.device("cpu")
+ self.max_obj = torch.tensor([max_obj]).to(device)
+ self.iou_threshold = torch.tensor([iou_thres]).to(device)
+ self.score_threshold = torch.tensor([score_thres]).to(device)
+ self.max_wh = max_wh # if max_wh != 0 : non-agnostic else : agnostic
+ self.convert_matrix = torch.tensor([[1, 0, 1, 0], [0, 1, 0, 1], [-0.5, 0, 0.5, 0], [0, -0.5, 0, 0.5]],
+ dtype=torch.float32,
+ device=self.device)
+ self.n_classes=n_classes
+
+ def forward(self, x):
+ ## https://github.com/thaitc-hust/yolov9-tensorrt/blob/main/torch2onnx.py
+ ## thanks https://github.com/thaitc-hust
+ if isinstance(x, list): ## yolov9-c.pt and yolov9-e.pt return list
+ x = x[1]
+ x = x.permute(0, 2, 1)
+ bboxes_x = x[..., 0:1]
+ bboxes_y = x[..., 1:2]
+ bboxes_w = x[..., 2:3]
+ bboxes_h = x[..., 3:4]
+ bboxes = torch.cat([bboxes_x, bboxes_y, bboxes_w, bboxes_h], dim = -1)
+ bboxes = bboxes.unsqueeze(2) # [n_batch, n_bboxes, 4] -> [n_batch, n_bboxes, 1, 4]
+ obj_conf = x[..., 4:]
+ scores = obj_conf
+ bboxes @= self.convert_matrix
+ max_score, category_id = scores.max(2, keepdim=True)
+ dis = category_id.float() * self.max_wh
+ nmsbox = bboxes + dis
+ max_score_tp = max_score.transpose(1, 2).contiguous()
+ selected_indices = ORT_NMS.apply(nmsbox, max_score_tp, self.max_obj, self.iou_threshold, self.score_threshold)
+ X, Y = selected_indices[:, 0], selected_indices[:, 2]
+ selected_boxes = bboxes[X, Y, :]
+ selected_categories = category_id[X, Y, :].float()
+ selected_scores = max_score[X, Y, :]
+ X = X.unsqueeze(1).float()
+ return torch.cat([X, selected_boxes, selected_categories, selected_scores], 1)
+
+
+class ONNX_TRT(nn.Module):
+ '''onnx module with TensorRT NMS operation.'''
+ def __init__(self, max_obj=100, iou_thres=0.45, score_thres=0.25, max_wh=None ,device=None, n_classes=80):
+ super().__init__()
+ assert max_wh is None
+ self.device = device if device else torch.device('cpu')
+ self.background_class = -1,
+ self.box_coding = 1,
+ self.iou_threshold = iou_thres
+ self.max_obj = max_obj
+ self.plugin_version = '1'
+ self.score_activation = 0
+ self.score_threshold = score_thres
+ self.n_classes=n_classes
+
+ def forward(self, x):
+ ## https://github.com/thaitc-hust/yolov9-tensorrt/blob/main/torch2onnx.py
+ ## thanks https://github.com/thaitc-hust
+ if isinstance(x, list): ## yolov9-c.pt and yolov9-e.pt return list
+ x = x[1]
+ x = x.permute(0, 2, 1)
+ bboxes_x = x[..., 0:1]
+ bboxes_y = x[..., 1:2]
+ bboxes_w = x[..., 2:3]
+ bboxes_h = x[..., 3:4]
+ bboxes = torch.cat([bboxes_x, bboxes_y, bboxes_w, bboxes_h], dim = -1)
+ bboxes = bboxes.unsqueeze(2) # [n_batch, n_bboxes, 4] -> [n_batch, n_bboxes, 1, 4]
+ obj_conf = x[..., 4:]
+ scores = obj_conf
+ num_det, det_boxes, det_scores, det_classes = TRT_NMS.apply(bboxes, scores, self.background_class, self.box_coding,
+ self.iou_threshold, self.max_obj,
+ self.plugin_version, self.score_activation,
+ self.score_threshold)
+ return num_det, det_boxes, det_scores, det_classes
+
+class End2End(nn.Module):
+ '''export onnx or tensorrt model with NMS operation.'''
+ def __init__(self, model, max_obj=100, iou_thres=0.45, score_thres=0.25, max_wh=None, device=None, n_classes=80):
+ super().__init__()
+ device = device if device else torch.device('cpu')
+ assert isinstance(max_wh,(int)) or max_wh is None
+ self.model = model.to(device)
+ self.model.model[-1].end2end = True
+ self.patch_model = ONNX_TRT if max_wh is None else ONNX_ORT
+ self.end2end = self.patch_model(max_obj, iou_thres, score_thres, max_wh, device, n_classes)
+ self.end2end.eval()
+
+ def forward(self, x):
+ x = self.model(x)
+ x = self.end2end(x)
+ return x
+
+
+def attempt_load(weights, device=None, inplace=True, fuse=True):
+ # Loads an ensemble of models weights=[a,b,c] or a single model weights=[a] or weights=a
+ from models.yolo import Detect, Model
+
+ model = Ensemble()
+ for w in weights if isinstance(weights, list) else [weights]:
+ ckpt = torch.load(attempt_download(w), map_location='cpu') # load
+ ckpt = (ckpt.get('ema') or ckpt['model']).to(device).float() # FP32 model
+
+ # Model compatibility updates
+ if not hasattr(ckpt, 'stride'):
+ ckpt.stride = torch.tensor([32.])
+ if hasattr(ckpt, 'names') and isinstance(ckpt.names, (list, tuple)):
+ ckpt.names = dict(enumerate(ckpt.names)) # convert to dict
+
+ model.append(ckpt.fuse().eval() if fuse and hasattr(ckpt, 'fuse') else ckpt.eval()) # model in eval mode
+
+ # Module compatibility updates
+ for m in model.modules():
+ t = type(m)
+ if t in (nn.Hardswish, nn.LeakyReLU, nn.ReLU, nn.ReLU6, nn.SiLU, Detect, Model):
+ m.inplace = inplace # torch 1.7.0 compatibility
+ # if t is Detect and not isinstance(m.anchor_grid, list):
+ # delattr(m, 'anchor_grid')
+ # setattr(m, 'anchor_grid', [torch.zeros(1)] * m.nl)
+ elif t is nn.Upsample and not hasattr(m, 'recompute_scale_factor'):
+ m.recompute_scale_factor = None # torch 1.11.0 compatibility
+
+ # Return model
+ if len(model) == 1:
+ return model[-1]
+
+ # Return detection ensemble
+ print(f'Ensemble created with {weights}\n')
+ for k in 'names', 'nc', 'yaml':
+ setattr(model, k, getattr(model[0], k))
+ model.stride = model[torch.argmax(torch.tensor([m.stride.max() for m in model])).int()].stride # max stride
+ assert all(model[0].nc == m.nc for m in model), f'Models have different class counts: {[m.nc for m in model]}'
+ return model
diff --git a/models/hub/anchors.yaml b/models/hub/anchors.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..65e85cf4764a30aa98abcc7f9daefdebbe94b29e
--- /dev/null
+++ b/models/hub/anchors.yaml
@@ -0,0 +1,59 @@
+# YOLOv3 & YOLOv5
+# Default anchors for COCO data
+
+
+# P5 -------------------------------------------------------------------------------------------------------------------
+# P5-640:
+anchors_p5_640:
+ - [10,13, 16,30, 33,23] # P3/8
+ - [30,61, 62,45, 59,119] # P4/16
+ - [116,90, 156,198, 373,326] # P5/32
+
+
+# P6 -------------------------------------------------------------------------------------------------------------------
+# P6-640: thr=0.25: 0.9964 BPR, 5.54 anchors past thr, n=12, img_size=640, metric_all=0.281/0.716-mean/best, past_thr=0.469-mean: 9,11, 21,19, 17,41, 43,32, 39,70, 86,64, 65,131, 134,130, 120,265, 282,180, 247,354, 512,387
+anchors_p6_640:
+ - [9,11, 21,19, 17,41] # P3/8
+ - [43,32, 39,70, 86,64] # P4/16
+ - [65,131, 134,130, 120,265] # P5/32
+ - [282,180, 247,354, 512,387] # P6/64
+
+# P6-1280: thr=0.25: 0.9950 BPR, 5.55 anchors past thr, n=12, img_size=1280, metric_all=0.281/0.714-mean/best, past_thr=0.468-mean: 19,27, 44,40, 38,94, 96,68, 86,152, 180,137, 140,301, 303,264, 238,542, 436,615, 739,380, 925,792
+anchors_p6_1280:
+ - [19,27, 44,40, 38,94] # P3/8
+ - [96,68, 86,152, 180,137] # P4/16
+ - [140,301, 303,264, 238,542] # P5/32
+ - [436,615, 739,380, 925,792] # P6/64
+
+# P6-1920: thr=0.25: 0.9950 BPR, 5.55 anchors past thr, n=12, img_size=1920, metric_all=0.281/0.714-mean/best, past_thr=0.468-mean: 28,41, 67,59, 57,141, 144,103, 129,227, 270,205, 209,452, 455,396, 358,812, 653,922, 1109,570, 1387,1187
+anchors_p6_1920:
+ - [28,41, 67,59, 57,141] # P3/8
+ - [144,103, 129,227, 270,205] # P4/16
+ - [209,452, 455,396, 358,812] # P5/32
+ - [653,922, 1109,570, 1387,1187] # P6/64
+
+
+# P7 -------------------------------------------------------------------------------------------------------------------
+# P7-640: thr=0.25: 0.9962 BPR, 6.76 anchors past thr, n=15, img_size=640, metric_all=0.275/0.733-mean/best, past_thr=0.466-mean: 11,11, 13,30, 29,20, 30,46, 61,38, 39,92, 78,80, 146,66, 79,163, 149,150, 321,143, 157,303, 257,402, 359,290, 524,372
+anchors_p7_640:
+ - [11,11, 13,30, 29,20] # P3/8
+ - [30,46, 61,38, 39,92] # P4/16
+ - [78,80, 146,66, 79,163] # P5/32
+ - [149,150, 321,143, 157,303] # P6/64
+ - [257,402, 359,290, 524,372] # P7/128
+
+# P7-1280: thr=0.25: 0.9968 BPR, 6.71 anchors past thr, n=15, img_size=1280, metric_all=0.273/0.732-mean/best, past_thr=0.463-mean: 19,22, 54,36, 32,77, 70,83, 138,71, 75,173, 165,159, 148,334, 375,151, 334,317, 251,626, 499,474, 750,326, 534,814, 1079,818
+anchors_p7_1280:
+ - [19,22, 54,36, 32,77] # P3/8
+ - [70,83, 138,71, 75,173] # P4/16
+ - [165,159, 148,334, 375,151] # P5/32
+ - [334,317, 251,626, 499,474] # P6/64
+ - [750,326, 534,814, 1079,818] # P7/128
+
+# P7-1920: thr=0.25: 0.9968 BPR, 6.71 anchors past thr, n=15, img_size=1920, metric_all=0.273/0.732-mean/best, past_thr=0.463-mean: 29,34, 81,55, 47,115, 105,124, 207,107, 113,259, 247,238, 222,500, 563,227, 501,476, 376,939, 749,711, 1126,489, 801,1222, 1618,1227
+anchors_p7_1920:
+ - [29,34, 81,55, 47,115] # P3/8
+ - [105,124, 207,107, 113,259] # P4/16
+ - [247,238, 222,500, 563,227] # P5/32
+ - [501,476, 376,939, 749,711] # P6/64
+ - [1126,489, 801,1222, 1618,1227] # P7/128
diff --git a/models/hub/yolov3-spp.yaml b/models/hub/yolov3-spp.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..6fb1c0b72a99d7ef26211060045d378e814fac50
--- /dev/null
+++ b/models/hub/yolov3-spp.yaml
@@ -0,0 +1,51 @@
+# YOLOv3
+
+# Parameters
+nc: 80 # number of classes
+depth_multiple: 1.0 # model depth multiple
+width_multiple: 1.0 # layer channel multiple
+anchors:
+ - [10,13, 16,30, 33,23] # P3/8
+ - [30,61, 62,45, 59,119] # P4/16
+ - [116,90, 156,198, 373,326] # P5/32
+
+# darknet53 backbone
+backbone:
+ # [from, number, module, args]
+ [[-1, 1, Conv, [32, 3, 1]], # 0
+ [-1, 1, Conv, [64, 3, 2]], # 1-P1/2
+ [-1, 1, Bottleneck, [64]],
+ [-1, 1, Conv, [128, 3, 2]], # 3-P2/4
+ [-1, 2, Bottleneck, [128]],
+ [-1, 1, Conv, [256, 3, 2]], # 5-P3/8
+ [-1, 8, Bottleneck, [256]],
+ [-1, 1, Conv, [512, 3, 2]], # 7-P4/16
+ [-1, 8, Bottleneck, [512]],
+ [-1, 1, Conv, [1024, 3, 2]], # 9-P5/32
+ [-1, 4, Bottleneck, [1024]], # 10
+ ]
+
+# YOLOv3-SPP head
+head:
+ [[-1, 1, Bottleneck, [1024, False]],
+ [-1, 1, SPP, [512, [5, 9, 13]]],
+ [-1, 1, Conv, [1024, 3, 1]],
+ [-1, 1, Conv, [512, 1, 1]],
+ [-1, 1, Conv, [1024, 3, 1]], # 15 (P5/32-large)
+
+ [-2, 1, Conv, [256, 1, 1]],
+ [-1, 1, nn.Upsample, [None, 2, 'nearest']],
+ [[-1, 8], 1, Concat, [1]], # cat backbone P4
+ [-1, 1, Bottleneck, [512, False]],
+ [-1, 1, Bottleneck, [512, False]],
+ [-1, 1, Conv, [256, 1, 1]],
+ [-1, 1, Conv, [512, 3, 1]], # 22 (P4/16-medium)
+
+ [-2, 1, Conv, [128, 1, 1]],
+ [-1, 1, nn.Upsample, [None, 2, 'nearest']],
+ [[-1, 6], 1, Concat, [1]], # cat backbone P3
+ [-1, 1, Bottleneck, [256, False]],
+ [-1, 2, Bottleneck, [256, False]], # 27 (P3/8-small)
+
+ [[27, 22, 15], 1, Detect, [nc, anchors]], # Detect(P3, P4, P5)
+ ]
diff --git a/models/hub/yolov3-tiny.yaml b/models/hub/yolov3-tiny.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..47372e09ae99ed6f0d494cc7811568c109b3f721
--- /dev/null
+++ b/models/hub/yolov3-tiny.yaml
@@ -0,0 +1,41 @@
+# YOLOv3
+
+# Parameters
+nc: 80 # number of classes
+depth_multiple: 1.0 # model depth multiple
+width_multiple: 1.0 # layer channel multiple
+anchors:
+ - [10,14, 23,27, 37,58] # P4/16
+ - [81,82, 135,169, 344,319] # P5/32
+
+# YOLOv3-tiny backbone
+backbone:
+ # [from, number, module, args]
+ [[-1, 1, Conv, [16, 3, 1]], # 0
+ [-1, 1, nn.MaxPool2d, [2, 2, 0]], # 1-P1/2
+ [-1, 1, Conv, [32, 3, 1]],
+ [-1, 1, nn.MaxPool2d, [2, 2, 0]], # 3-P2/4
+ [-1, 1, Conv, [64, 3, 1]],
+ [-1, 1, nn.MaxPool2d, [2, 2, 0]], # 5-P3/8
+ [-1, 1, Conv, [128, 3, 1]],
+ [-1, 1, nn.MaxPool2d, [2, 2, 0]], # 7-P4/16
+ [-1, 1, Conv, [256, 3, 1]],
+ [-1, 1, nn.MaxPool2d, [2, 2, 0]], # 9-P5/32
+ [-1, 1, Conv, [512, 3, 1]],
+ [-1, 1, nn.ZeroPad2d, [[0, 1, 0, 1]]], # 11
+ [-1, 1, nn.MaxPool2d, [2, 1, 0]], # 12
+ ]
+
+# YOLOv3-tiny head
+head:
+ [[-1, 1, Conv, [1024, 3, 1]],
+ [-1, 1, Conv, [256, 1, 1]],
+ [-1, 1, Conv, [512, 3, 1]], # 15 (P5/32-large)
+
+ [-2, 1, Conv, [128, 1, 1]],
+ [-1, 1, nn.Upsample, [None, 2, 'nearest']],
+ [[-1, 8], 1, Concat, [1]], # cat backbone P4
+ [-1, 1, Conv, [256, 3, 1]], # 19 (P4/16-medium)
+
+ [[19, 15], 1, Detect, [nc, anchors]], # Detect(P4, P5)
+ ]
diff --git a/models/hub/yolov3.yaml b/models/hub/yolov3.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..3ebd78f5eb66243ca6f0235904d4df9ef05d5f36
--- /dev/null
+++ b/models/hub/yolov3.yaml
@@ -0,0 +1,51 @@
+# YOLOv3
+
+# Parameters
+nc: 80 # number of classes
+depth_multiple: 1.0 # model depth multiple
+width_multiple: 1.0 # layer channel multiple
+anchors:
+ - [10,13, 16,30, 33,23] # P3/8
+ - [30,61, 62,45, 59,119] # P4/16
+ - [116,90, 156,198, 373,326] # P5/32
+
+# darknet53 backbone
+backbone:
+ # [from, number, module, args]
+ [[-1, 1, Conv, [32, 3, 1]], # 0
+ [-1, 1, Conv, [64, 3, 2]], # 1-P1/2
+ [-1, 1, Bottleneck, [64]],
+ [-1, 1, Conv, [128, 3, 2]], # 3-P2/4
+ [-1, 2, Bottleneck, [128]],
+ [-1, 1, Conv, [256, 3, 2]], # 5-P3/8
+ [-1, 8, Bottleneck, [256]],
+ [-1, 1, Conv, [512, 3, 2]], # 7-P4/16
+ [-1, 8, Bottleneck, [512]],
+ [-1, 1, Conv, [1024, 3, 2]], # 9-P5/32
+ [-1, 4, Bottleneck, [1024]], # 10
+ ]
+
+# YOLOv3 head
+head:
+ [[-1, 1, Bottleneck, [1024, False]],
+ [-1, 1, Conv, [512, 1, 1]],
+ [-1, 1, Conv, [1024, 3, 1]],
+ [-1, 1, Conv, [512, 1, 1]],
+ [-1, 1, Conv, [1024, 3, 1]], # 15 (P5/32-large)
+
+ [-2, 1, Conv, [256, 1, 1]],
+ [-1, 1, nn.Upsample, [None, 2, 'nearest']],
+ [[-1, 8], 1, Concat, [1]], # cat backbone P4
+ [-1, 1, Bottleneck, [512, False]],
+ [-1, 1, Bottleneck, [512, False]],
+ [-1, 1, Conv, [256, 1, 1]],
+ [-1, 1, Conv, [512, 3, 1]], # 22 (P4/16-medium)
+
+ [-2, 1, Conv, [128, 1, 1]],
+ [-1, 1, nn.Upsample, [None, 2, 'nearest']],
+ [[-1, 6], 1, Concat, [1]], # cat backbone P3
+ [-1, 1, Bottleneck, [256, False]],
+ [-1, 2, Bottleneck, [256, False]], # 27 (P3/8-small)
+
+ [[27, 22, 15], 1, Detect, [nc, anchors]], # Detect(P3, P4, P5)
+ ]
diff --git a/models/panoptic/gelan-c-pan.yaml b/models/panoptic/gelan-c-pan.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..acc41c4e0230b2deecfbab9f8e83fe1b00a1e4a7
--- /dev/null
+++ b/models/panoptic/gelan-c-pan.yaml
@@ -0,0 +1,80 @@
+# YOLOv9
+
+# parameters
+nc: 80 # number of classes
+depth_multiple: 1.0 # model depth multiple
+width_multiple: 1.0 # layer channel multiple
+#activation: nn.LeakyReLU(0.1)
+#activation: nn.ReLU()
+
+# anchors
+anchors: 3
+
+# gelan backbone
+backbone:
+ [
+ # conv down
+ [-1, 1, Conv, [64, 3, 2]], # 0-P1/2
+
+ # conv down
+ [-1, 1, Conv, [128, 3, 2]], # 1-P2/4
+
+ # elan-1 block
+ [-1, 1, RepNCSPELAN4, [256, 128, 64, 1]], # 2
+
+ # avg-conv down
+ [-1, 1, ADown, [256]], # 3-P3/8
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 256, 128, 1]], # 4
+
+ # avg-conv down
+ [-1, 1, ADown, [512]], # 5-P4/16
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 512, 256, 1]], # 6
+
+ # avg-conv down
+ [-1, 1, ADown, [512]], # 7-P5/32
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 512, 256, 1]], # 8
+ ]
+
+# gelan head
+head:
+ [
+ # elan-spp block
+ [-1, 1, SPPELAN, [512, 256]], # 9
+
+ # up-concat merge
+ [-1, 1, nn.Upsample, [None, 2, 'nearest']],
+ [[-1, 6], 1, Concat, [1]], # cat backbone P4
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 512, 256, 1]], # 12
+
+ # up-concat merge
+ [-1, 1, nn.Upsample, [None, 2, 'nearest']],
+ [[-1, 4], 1, Concat, [1]], # cat backbone P3
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [256, 256, 128, 1]], # 15 (P3/8-small)
+
+ # avg-conv-down merge
+ [-1, 1, ADown, [256]],
+ [[-1, 12], 1, Concat, [1]], # cat head P4
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 512, 256, 1]], # 18 (P4/16-medium)
+
+ # avg-conv-down merge
+ [-1, 1, ADown, [512]],
+ [[-1, 9], 1, Concat, [1]], # cat head P5
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 512, 256, 1]], # 21 (P5/32-large)
+
+ # panoptic
+ [[15, 18, 21], 1, Panoptic, [nc, 93, 32, 256]], # Panoptic(P3, P4, P5)
+ ]
diff --git a/models/panoptic/yolov7-af-pan.yaml b/models/panoptic/yolov7-af-pan.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..a9bed1d1dda4bfd0415731a999b99c946e35e056
--- /dev/null
+++ b/models/panoptic/yolov7-af-pan.yaml
@@ -0,0 +1,137 @@
+# YOLOv7
+
+# Parameters
+nc: 80 # number of classes
+sem_nc: 93 # number of stuff classes
+depth_multiple: 1.0 # model depth multiple
+width_multiple: 1.0 # layer channel multiple
+anchors: 3
+
+# YOLOv7 backbone
+backbone:
+ [[-1, 1, Conv, [32, 3, 1]], # 0
+
+ [-1, 1, Conv, [64, 3, 2]], # 1-P1/2
+ [-1, 1, Conv, [64, 3, 1]],
+
+ [-1, 1, Conv, [128, 3, 2]], # 3-P2/4
+ [-1, 1, Conv, [64, 1, 1]],
+ [-2, 1, Conv, [64, 1, 1]],
+ [-1, 1, Conv, [64, 3, 1]],
+ [-1, 1, Conv, [64, 3, 1]],
+ [-1, 1, Conv, [64, 3, 1]],
+ [-1, 1, Conv, [64, 3, 1]],
+ [[-1, -3, -5, -6], 1, Concat, [1]],
+ [-1, 1, Conv, [256, 1, 1]], # 11
+
+ [-1, 1, MP, []],
+ [-1, 1, Conv, [128, 1, 1]],
+ [-3, 1, Conv, [128, 1, 1]],
+ [-1, 1, Conv, [128, 3, 2]],
+ [[-1, -3], 1, Concat, [1]], # 16-P3/8
+ [-1, 1, Conv, [128, 1, 1]],
+ [-2, 1, Conv, [128, 1, 1]],
+ [-1, 1, Conv, [128, 3, 1]],
+ [-1, 1, Conv, [128, 3, 1]],
+ [-1, 1, Conv, [128, 3, 1]],
+ [-1, 1, Conv, [128, 3, 1]],
+ [[-1, -3, -5, -6], 1, Concat, [1]],
+ [-1, 1, Conv, [512, 1, 1]], # 24
+
+ [-1, 1, MP, []],
+ [-1, 1, Conv, [256, 1, 1]],
+ [-3, 1, Conv, [256, 1, 1]],
+ [-1, 1, Conv, [256, 3, 2]],
+ [[-1, -3], 1, Concat, [1]], # 29-P4/16
+ [-1, 1, Conv, [256, 1, 1]],
+ [-2, 1, Conv, [256, 1, 1]],
+ [-1, 1, Conv, [256, 3, 1]],
+ [-1, 1, Conv, [256, 3, 1]],
+ [-1, 1, Conv, [256, 3, 1]],
+ [-1, 1, Conv, [256, 3, 1]],
+ [[-1, -3, -5, -6], 1, Concat, [1]],
+ [-1, 1, Conv, [1024, 1, 1]], # 37
+
+ [-1, 1, MP, []],
+ [-1, 1, Conv, [512, 1, 1]],
+ [-3, 1, Conv, [512, 1, 1]],
+ [-1, 1, Conv, [512, 3, 2]],
+ [[-1, -3], 1, Concat, [1]], # 42-P5/32
+ [-1, 1, Conv, [256, 1, 1]],
+ [-2, 1, Conv, [256, 1, 1]],
+ [-1, 1, Conv, [256, 3, 1]],
+ [-1, 1, Conv, [256, 3, 1]],
+ [-1, 1, Conv, [256, 3, 1]],
+ [-1, 1, Conv, [256, 3, 1]],
+ [[-1, -3, -5, -6], 1, Concat, [1]],
+ [-1, 1, Conv, [1024, 1, 1]], # 50
+ ]
+
+# yolov7 head
+head:
+ [[-1, 1, SPPCSPC, [512]], # 51
+
+ [-1, 1, Conv, [256, 1, 1]],
+ [-1, 1, nn.Upsample, [None, 2, 'nearest']],
+ [37, 1, Conv, [256, 1, 1]], # route backbone P4
+ [[-1, -2], 1, Concat, [1]],
+
+ [-1, 1, Conv, [256, 1, 1]],
+ [-2, 1, Conv, [256, 1, 1]],
+ [-1, 1, Conv, [128, 3, 1]],
+ [-1, 1, Conv, [128, 3, 1]],
+ [-1, 1, Conv, [128, 3, 1]],
+ [-1, 1, Conv, [128, 3, 1]],
+ [[-1, -2, -3, -4, -5, -6], 1, Concat, [1]],
+ [-1, 1, Conv, [256, 1, 1]], # 63
+
+ [-1, 1, Conv, [128, 1, 1]],
+ [-1, 1, nn.Upsample, [None, 2, 'nearest']],
+ [24, 1, Conv, [128, 1, 1]], # route backbone P3
+ [[-1, -2], 1, Concat, [1]],
+
+ [-1, 1, Conv, [128, 1, 1]],
+ [-2, 1, Conv, [128, 1, 1]],
+ [-1, 1, Conv, [64, 3, 1]],
+ [-1, 1, Conv, [64, 3, 1]],
+ [-1, 1, Conv, [64, 3, 1]],
+ [-1, 1, Conv, [64, 3, 1]],
+ [[-1, -2, -3, -4, -5, -6], 1, Concat, [1]],
+ [-1, 1, Conv, [128, 1, 1]], # 75
+
+ [-1, 1, MP, []],
+ [-1, 1, Conv, [128, 1, 1]],
+ [-3, 1, Conv, [128, 1, 1]],
+ [-1, 1, Conv, [128, 3, 2]],
+ [[-1, -3, 63], 1, Concat, [1]],
+
+ [-1, 1, Conv, [256, 1, 1]],
+ [-2, 1, Conv, [256, 1, 1]],
+ [-1, 1, Conv, [128, 3, 1]],
+ [-1, 1, Conv, [128, 3, 1]],
+ [-1, 1, Conv, [128, 3, 1]],
+ [-1, 1, Conv, [128, 3, 1]],
+ [[-1, -2, -3, -4, -5, -6], 1, Concat, [1]],
+ [-1, 1, Conv, [256, 1, 1]], # 88
+
+ [-1, 1, MP, []],
+ [-1, 1, Conv, [256, 1, 1]],
+ [-3, 1, Conv, [256, 1, 1]],
+ [-1, 1, Conv, [256, 3, 2]],
+ [[-1, -3, 51], 1, Concat, [1]],
+
+ [-1, 1, Conv, [512, 1, 1]],
+ [-2, 1, Conv, [512, 1, 1]],
+ [-1, 1, Conv, [256, 3, 1]],
+ [-1, 1, Conv, [256, 3, 1]],
+ [-1, 1, Conv, [256, 3, 1]],
+ [-1, 1, Conv, [256, 3, 1]],
+ [[-1, -2, -3, -4, -5, -6], 1, Concat, [1]],
+ [-1, 1, Conv, [512, 1, 1]], # 101
+
+ [75, 1, Conv, [256, 3, 1]],
+ [88, 1, Conv, [512, 3, 1]],
+ [101, 1, Conv, [1024, 3, 1]],
+
+ [[102, 103, 104], 1, Panoptic, [nc, 93, 32, 256]], # Panoptic(P3, P4, P5)
+ ]
diff --git a/models/segment/gelan-c-dseg.yaml b/models/segment/gelan-c-dseg.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..8e4a8e839958859ec11414926e58913737a50d15
--- /dev/null
+++ b/models/segment/gelan-c-dseg.yaml
@@ -0,0 +1,84 @@
+# YOLOv9
+
+# parameters
+nc: 80 # number of classes
+depth_multiple: 1.0 # model depth multiple
+width_multiple: 1.0 # layer channel multiple
+#activation: nn.LeakyReLU(0.1)
+#activation: nn.ReLU()
+
+# anchors
+anchors: 3
+
+# gelan backbone
+backbone:
+ [
+ # conv down
+ [-1, 1, Conv, [64, 3, 2]], # 0-P1/2
+
+ # conv down
+ [-1, 1, Conv, [128, 3, 2]], # 1-P2/4
+
+ # elan-1 block
+ [-1, 1, RepNCSPELAN4, [256, 128, 64, 1]], # 2
+
+ # avg-conv down
+ [-1, 1, ADown, [256]], # 3-P3/8
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 256, 128, 1]], # 4
+
+ # avg-conv down
+ [-1, 1, ADown, [512]], # 5-P4/16
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 512, 256, 1]], # 6
+
+ # avg-conv down
+ [-1, 1, ADown, [512]], # 7-P5/32
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 512, 256, 1]], # 8
+ ]
+
+# gelan head
+head:
+ [
+ # elan-spp block
+ [-1, 1, SPPELAN, [512, 256]], # 9
+
+ # up-concat merge
+ [-1, 1, nn.Upsample, [None, 2, 'nearest']],
+ [[-1, 6], 1, Concat, [1]], # cat backbone P4
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 512, 256, 1]], # 12
+
+ # up-concat merge
+ [-1, 1, nn.Upsample, [None, 2, 'nearest']],
+ [[-1, 4], 1, Concat, [1]], # cat backbone P3
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [256, 256, 128, 1]], # 15 (P3/8-small)
+
+ # avg-conv-down merge
+ [-1, 1, ADown, [256]],
+ [[-1, 12], 1, Concat, [1]], # cat head P4
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 512, 256, 1]], # 18 (P4/16-medium)
+
+ # avg-conv-down merge
+ [-1, 1, ADown, [512]],
+ [[-1, 9], 1, Concat, [1]], # cat head P5
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 512, 256, 1]], # 21 (P5/32-large)
+
+ [15, 1, RepNCSPELAN4, [256, 256, 128, 1]], # 22
+ [-1, 1, nn.Upsample, [None, 2, 'nearest']],
+ [-1, 1, Conv, [256, 3, 1]], # 24
+
+ # segment
+ [[15, 18, 21, 24], 1, DSegment, [nc, 32, 256]], # Segment(P3, P4, P5)
+ ]
diff --git a/models/segment/gelan-c-seg.yaml b/models/segment/gelan-c-seg.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..d7815bb3db89c5b2a4fdf1e2bef44ff9b23fc923
--- /dev/null
+++ b/models/segment/gelan-c-seg.yaml
@@ -0,0 +1,80 @@
+# YOLOv9
+
+# parameters
+nc: 80 # number of classes
+depth_multiple: 1.0 # model depth multiple
+width_multiple: 1.0 # layer channel multiple
+#activation: nn.LeakyReLU(0.1)
+#activation: nn.ReLU()
+
+# anchors
+anchors: 3
+
+# gelan backbone
+backbone:
+ [
+ # conv down
+ [-1, 1, Conv, [64, 3, 2]], # 0-P1/2
+
+ # conv down
+ [-1, 1, Conv, [128, 3, 2]], # 1-P2/4
+
+ # elan-1 block
+ [-1, 1, RepNCSPELAN4, [256, 128, 64, 1]], # 2
+
+ # avg-conv down
+ [-1, 1, ADown, [256]], # 3-P3/8
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 256, 128, 1]], # 4
+
+ # avg-conv down
+ [-1, 1, ADown, [512]], # 5-P4/16
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 512, 256, 1]], # 6
+
+ # avg-conv down
+ [-1, 1, ADown, [512]], # 7-P5/32
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 512, 256, 1]], # 8
+ ]
+
+# gelan head
+head:
+ [
+ # elan-spp block
+ [-1, 1, SPPELAN, [512, 256]], # 9
+
+ # up-concat merge
+ [-1, 1, nn.Upsample, [None, 2, 'nearest']],
+ [[-1, 6], 1, Concat, [1]], # cat backbone P4
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 512, 256, 1]], # 12
+
+ # up-concat merge
+ [-1, 1, nn.Upsample, [None, 2, 'nearest']],
+ [[-1, 4], 1, Concat, [1]], # cat backbone P3
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [256, 256, 128, 1]], # 15 (P3/8-small)
+
+ # avg-conv-down merge
+ [-1, 1, ADown, [256]],
+ [[-1, 12], 1, Concat, [1]], # cat head P4
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 512, 256, 1]], # 18 (P4/16-medium)
+
+ # avg-conv-down merge
+ [-1, 1, ADown, [512]],
+ [[-1, 9], 1, Concat, [1]], # cat head P5
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 512, 256, 1]], # 21 (P5/32-large)
+
+ # segment
+ [[15, 18, 21], 1, Segment, [nc, 32, 256]], # Segment(P3, P4, P5)
+ ]
diff --git a/models/segment/yolov7-af-seg.yaml b/models/segment/yolov7-af-seg.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..0e4b61f7b65377ea9ec40595c9034454be431341
--- /dev/null
+++ b/models/segment/yolov7-af-seg.yaml
@@ -0,0 +1,136 @@
+# YOLOv7
+
+# Parameters
+nc: 80 # number of classes
+depth_multiple: 1.0 # model depth multiple
+width_multiple: 1.0 # layer channel multiple
+anchors: 3
+
+# YOLOv7 backbone
+backbone:
+ [[-1, 1, Conv, [32, 3, 1]], # 0
+
+ [-1, 1, Conv, [64, 3, 2]], # 1-P1/2
+ [-1, 1, Conv, [64, 3, 1]],
+
+ [-1, 1, Conv, [128, 3, 2]], # 3-P2/4
+ [-1, 1, Conv, [64, 1, 1]],
+ [-2, 1, Conv, [64, 1, 1]],
+ [-1, 1, Conv, [64, 3, 1]],
+ [-1, 1, Conv, [64, 3, 1]],
+ [-1, 1, Conv, [64, 3, 1]],
+ [-1, 1, Conv, [64, 3, 1]],
+ [[-1, -3, -5, -6], 1, Concat, [1]],
+ [-1, 1, Conv, [256, 1, 1]], # 11
+
+ [-1, 1, MP, []],
+ [-1, 1, Conv, [128, 1, 1]],
+ [-3, 1, Conv, [128, 1, 1]],
+ [-1, 1, Conv, [128, 3, 2]],
+ [[-1, -3], 1, Concat, [1]], # 16-P3/8
+ [-1, 1, Conv, [128, 1, 1]],
+ [-2, 1, Conv, [128, 1, 1]],
+ [-1, 1, Conv, [128, 3, 1]],
+ [-1, 1, Conv, [128, 3, 1]],
+ [-1, 1, Conv, [128, 3, 1]],
+ [-1, 1, Conv, [128, 3, 1]],
+ [[-1, -3, -5, -6], 1, Concat, [1]],
+ [-1, 1, Conv, [512, 1, 1]], # 24
+
+ [-1, 1, MP, []],
+ [-1, 1, Conv, [256, 1, 1]],
+ [-3, 1, Conv, [256, 1, 1]],
+ [-1, 1, Conv, [256, 3, 2]],
+ [[-1, -3], 1, Concat, [1]], # 29-P4/16
+ [-1, 1, Conv, [256, 1, 1]],
+ [-2, 1, Conv, [256, 1, 1]],
+ [-1, 1, Conv, [256, 3, 1]],
+ [-1, 1, Conv, [256, 3, 1]],
+ [-1, 1, Conv, [256, 3, 1]],
+ [-1, 1, Conv, [256, 3, 1]],
+ [[-1, -3, -5, -6], 1, Concat, [1]],
+ [-1, 1, Conv, [1024, 1, 1]], # 37
+
+ [-1, 1, MP, []],
+ [-1, 1, Conv, [512, 1, 1]],
+ [-3, 1, Conv, [512, 1, 1]],
+ [-1, 1, Conv, [512, 3, 2]],
+ [[-1, -3], 1, Concat, [1]], # 42-P5/32
+ [-1, 1, Conv, [256, 1, 1]],
+ [-2, 1, Conv, [256, 1, 1]],
+ [-1, 1, Conv, [256, 3, 1]],
+ [-1, 1, Conv, [256, 3, 1]],
+ [-1, 1, Conv, [256, 3, 1]],
+ [-1, 1, Conv, [256, 3, 1]],
+ [[-1, -3, -5, -6], 1, Concat, [1]],
+ [-1, 1, Conv, [1024, 1, 1]], # 50
+ ]
+
+# yolov7 head
+head:
+ [[-1, 1, SPPCSPC, [512]], # 51
+
+ [-1, 1, Conv, [256, 1, 1]],
+ [-1, 1, nn.Upsample, [None, 2, 'nearest']],
+ [37, 1, Conv, [256, 1, 1]], # route backbone P4
+ [[-1, -2], 1, Concat, [1]],
+
+ [-1, 1, Conv, [256, 1, 1]],
+ [-2, 1, Conv, [256, 1, 1]],
+ [-1, 1, Conv, [128, 3, 1]],
+ [-1, 1, Conv, [128, 3, 1]],
+ [-1, 1, Conv, [128, 3, 1]],
+ [-1, 1, Conv, [128, 3, 1]],
+ [[-1, -2, -3, -4, -5, -6], 1, Concat, [1]],
+ [-1, 1, Conv, [256, 1, 1]], # 63
+
+ [-1, 1, Conv, [128, 1, 1]],
+ [-1, 1, nn.Upsample, [None, 2, 'nearest']],
+ [24, 1, Conv, [128, 1, 1]], # route backbone P3
+ [[-1, -2], 1, Concat, [1]],
+
+ [-1, 1, Conv, [128, 1, 1]],
+ [-2, 1, Conv, [128, 1, 1]],
+ [-1, 1, Conv, [64, 3, 1]],
+ [-1, 1, Conv, [64, 3, 1]],
+ [-1, 1, Conv, [64, 3, 1]],
+ [-1, 1, Conv, [64, 3, 1]],
+ [[-1, -2, -3, -4, -5, -6], 1, Concat, [1]],
+ [-1, 1, Conv, [128, 1, 1]], # 75
+
+ [-1, 1, MP, []],
+ [-1, 1, Conv, [128, 1, 1]],
+ [-3, 1, Conv, [128, 1, 1]],
+ [-1, 1, Conv, [128, 3, 2]],
+ [[-1, -3, 63], 1, Concat, [1]],
+
+ [-1, 1, Conv, [256, 1, 1]],
+ [-2, 1, Conv, [256, 1, 1]],
+ [-1, 1, Conv, [128, 3, 1]],
+ [-1, 1, Conv, [128, 3, 1]],
+ [-1, 1, Conv, [128, 3, 1]],
+ [-1, 1, Conv, [128, 3, 1]],
+ [[-1, -2, -3, -4, -5, -6], 1, Concat, [1]],
+ [-1, 1, Conv, [256, 1, 1]], # 88
+
+ [-1, 1, MP, []],
+ [-1, 1, Conv, [256, 1, 1]],
+ [-3, 1, Conv, [256, 1, 1]],
+ [-1, 1, Conv, [256, 3, 2]],
+ [[-1, -3, 51], 1, Concat, [1]],
+
+ [-1, 1, Conv, [512, 1, 1]],
+ [-2, 1, Conv, [512, 1, 1]],
+ [-1, 1, Conv, [256, 3, 1]],
+ [-1, 1, Conv, [256, 3, 1]],
+ [-1, 1, Conv, [256, 3, 1]],
+ [-1, 1, Conv, [256, 3, 1]],
+ [[-1, -2, -3, -4, -5, -6], 1, Concat, [1]],
+ [-1, 1, Conv, [512, 1, 1]], # 101
+
+ [75, 1, Conv, [256, 3, 1]],
+ [88, 1, Conv, [512, 3, 1]],
+ [101, 1, Conv, [1024, 3, 1]],
+
+ [[102, 103, 104], 1, Segment, [nc, 32, 256]], # Segment(P3, P4, P5)
+ ]
diff --git a/models/segment/yolov9-c-dseg.yaml b/models/segment/yolov9-c-dseg.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..44544511cb085d457e9b7909984d016473d2d687
--- /dev/null
+++ b/models/segment/yolov9-c-dseg.yaml
@@ -0,0 +1,130 @@
+# YOLOv9
+
+# parameters
+nc: 80 # number of classes
+depth_multiple: 1.0 # model depth multiple
+width_multiple: 1.0 # layer channel multiple
+#activation: nn.LeakyReLU(0.1)
+#activation: nn.ReLU()
+
+# anchors
+anchors: 3
+
+# gelan backbone
+backbone:
+ [
+ [-1, 1, Silence, []],
+
+ # conv down
+ [-1, 1, Conv, [64, 3, 2]], # 1-P1/2
+
+ # conv down
+ [-1, 1, Conv, [128, 3, 2]], # 2-P2/4
+
+ # elan-1 block
+ [-1, 1, RepNCSPELAN4, [256, 128, 64, 1]], # 3
+
+ # avg-conv down
+ [-1, 1, ADown, [256]], # 4-P3/8
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 256, 128, 1]], # 5
+
+ # avg-conv down
+ [-1, 1, ADown, [512]], # 6-P4/16
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 512, 256, 1]], # 7
+
+ # avg-conv down
+ [-1, 1, ADown, [512]], # 8-P5/32
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 512, 256, 1]], # 9
+ ]
+
+# YOLOv9 head
+head:
+ [
+ # elan-spp block
+ [-1, 1, SPPELAN, [512, 256]], # 10
+
+ # up-concat merge
+ [-1, 1, nn.Upsample, [None, 2, 'nearest']],
+ [[-1, 7], 1, Concat, [1]], # cat backbone P4
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 512, 256, 1]], # 13
+
+ # up-concat merge
+ [-1, 1, nn.Upsample, [None, 2, 'nearest']],
+ [[-1, 5], 1, Concat, [1]], # cat backbone P3
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [256, 256, 128, 1]], # 16 (P3/8-small)
+
+ # avg-conv-down merge
+ [-1, 1, ADown, [256]],
+ [[-1, 13], 1, Concat, [1]], # cat head P4
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 512, 256, 1]], # 19 (P4/16-medium)
+
+ # avg-conv-down merge
+ [-1, 1, ADown, [512]],
+ [[-1, 10], 1, Concat, [1]], # cat head P5
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 512, 256, 1]], # 22 (P5/32-large)
+
+
+ # multi-level reversible auxiliary branch
+
+ # routing
+ [5, 1, CBLinear, [[256]]], # 23
+ [7, 1, CBLinear, [[256, 512]]], # 24
+ [9, 1, CBLinear, [[256, 512, 512]]], # 25
+
+ # conv down
+ [0, 1, Conv, [64, 3, 2]], # 26-P1/2
+
+ # conv down
+ [-1, 1, Conv, [128, 3, 2]], # 27-P2/4
+
+ # elan-1 block
+ [-1, 1, RepNCSPELAN4, [256, 128, 64, 1]], # 28
+
+ # avg-conv down fuse
+ [-1, 1, ADown, [256]], # 29-P3/8
+ [[23, 24, 25, -1], 1, CBFuse, [[0, 0, 0]]], # 30
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 256, 128, 1]], # 31
+
+ # avg-conv down fuse
+ [-1, 1, ADown, [512]], # 32-P4/16
+ [[24, 25, -1], 1, CBFuse, [[1, 1]]], # 33
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 512, 256, 1]], # 34
+
+ # avg-conv down fuse
+ [-1, 1, ADown, [512]], # 35-P5/32
+ [[25, -1], 1, CBFuse, [[2]]], # 36
+
+ # elan-2 block
+ [-1, 1, RepNCSPELAN4, [512, 512, 256, 1]], # 37
+
+ [31, 1, RepNCSPELAN4, [512, 256, 128, 2]], # 38
+
+ [-1, 1, nn.Upsample, [None, 2, 'nearest']],
+ [-1, 1, Conv, [256, 3, 1]], # 40
+
+ [16, 1, RepNCSPELAN4, [256, 256, 128, 2]], # 41
+
+ [-1, 1, nn.Upsample, [None, 2, 'nearest']],
+ [-1, 1, Conv, [256, 3, 1]], # 43
+
+ # segment
+ [[31, 34, 37, 16, 19, 22, 40, 43], 1, DualDSegment, [nc, 32, 256]], # Segment(P3, P4, P5)
+ ]
diff --git a/models/tf.py b/models/tf.py
new file mode 100644
index 0000000000000000000000000000000000000000..897efafefebb6e176afd1f001ea14d9b922727e9
--- /dev/null
+++ b/models/tf.py
@@ -0,0 +1,596 @@
+import argparse
+import sys
+from copy import deepcopy
+from pathlib import Path
+
+FILE = Path(__file__).resolve()
+ROOT = FILE.parents[1] # YOLO root directory
+if str(ROOT) not in sys.path:
+ sys.path.append(str(ROOT)) # add ROOT to PATH
+# ROOT = ROOT.relative_to(Path.cwd()) # relative
+
+import numpy as np
+import tensorflow as tf
+import torch
+import torch.nn as nn
+from tensorflow import keras
+
+from models.common import (C3, SPP, SPPF, Bottleneck, BottleneckCSP, C3x, Concat, Conv, CrossConv, DWConv,
+ DWConvTranspose2d, Focus, autopad)
+from models.experimental import MixConv2d, attempt_load
+from models.yolo import Detect, Segment
+from utils.activations import SiLU
+from utils.general import LOGGER, make_divisible, print_args
+
+
+class TFBN(keras.layers.Layer):
+ # TensorFlow BatchNormalization wrapper
+ def __init__(self, w=None):
+ super().__init__()
+ self.bn = keras.layers.BatchNormalization(
+ beta_initializer=keras.initializers.Constant(w.bias.numpy()),
+ gamma_initializer=keras.initializers.Constant(w.weight.numpy()),
+ moving_mean_initializer=keras.initializers.Constant(w.running_mean.numpy()),
+ moving_variance_initializer=keras.initializers.Constant(w.running_var.numpy()),
+ epsilon=w.eps)
+
+ def call(self, inputs):
+ return self.bn(inputs)
+
+
+class TFPad(keras.layers.Layer):
+ # Pad inputs in spatial dimensions 1 and 2
+ def __init__(self, pad):
+ super().__init__()
+ if isinstance(pad, int):
+ self.pad = tf.constant([[0, 0], [pad, pad], [pad, pad], [0, 0]])
+ else: # tuple/list
+ self.pad = tf.constant([[0, 0], [pad[0], pad[0]], [pad[1], pad[1]], [0, 0]])
+
+ def call(self, inputs):
+ return tf.pad(inputs, self.pad, mode='constant', constant_values=0)
+
+
+class TFConv(keras.layers.Layer):
+ # Standard convolution
+ def __init__(self, c1, c2, k=1, s=1, p=None, g=1, act=True, w=None):
+ # ch_in, ch_out, weights, kernel, stride, padding, groups
+ super().__init__()
+ assert g == 1, "TF v2.2 Conv2D does not support 'groups' argument"
+ # TensorFlow convolution padding is inconsistent with PyTorch (e.g. k=3 s=2 'SAME' padding)
+ # see https://stackoverflow.com/questions/52975843/comparing-conv2d-with-padding-between-tensorflow-and-pytorch
+ conv = keras.layers.Conv2D(
+ filters=c2,
+ kernel_size=k,
+ strides=s,
+ padding='SAME' if s == 1 else 'VALID',
+ use_bias=not hasattr(w, 'bn'),
+ kernel_initializer=keras.initializers.Constant(w.conv.weight.permute(2, 3, 1, 0).numpy()),
+ bias_initializer='zeros' if hasattr(w, 'bn') else keras.initializers.Constant(w.conv.bias.numpy()))
+ self.conv = conv if s == 1 else keras.Sequential([TFPad(autopad(k, p)), conv])
+ self.bn = TFBN(w.bn) if hasattr(w, 'bn') else tf.identity
+ self.act = activations(w.act) if act else tf.identity
+
+ def call(self, inputs):
+ return self.act(self.bn(self.conv(inputs)))
+
+
+class TFDWConv(keras.layers.Layer):
+ # Depthwise convolution
+ def __init__(self, c1, c2, k=1, s=1, p=None, act=True, w=None):
+ # ch_in, ch_out, weights, kernel, stride, padding, groups
+ super().__init__()
+ assert c2 % c1 == 0, f'TFDWConv() output={c2} must be a multiple of input={c1} channels'
+ conv = keras.layers.DepthwiseConv2D(
+ kernel_size=k,
+ depth_multiplier=c2 // c1,
+ strides=s,
+ padding='SAME' if s == 1 else 'VALID',
+ use_bias=not hasattr(w, 'bn'),
+ depthwise_initializer=keras.initializers.Constant(w.conv.weight.permute(2, 3, 1, 0).numpy()),
+ bias_initializer='zeros' if hasattr(w, 'bn') else keras.initializers.Constant(w.conv.bias.numpy()))
+ self.conv = conv if s == 1 else keras.Sequential([TFPad(autopad(k, p)), conv])
+ self.bn = TFBN(w.bn) if hasattr(w, 'bn') else tf.identity
+ self.act = activations(w.act) if act else tf.identity
+
+ def call(self, inputs):
+ return self.act(self.bn(self.conv(inputs)))
+
+
+class TFDWConvTranspose2d(keras.layers.Layer):
+ # Depthwise ConvTranspose2d
+ def __init__(self, c1, c2, k=1, s=1, p1=0, p2=0, w=None):
+ # ch_in, ch_out, weights, kernel, stride, padding, groups
+ super().__init__()
+ assert c1 == c2, f'TFDWConv() output={c2} must be equal to input={c1} channels'
+ assert k == 4 and p1 == 1, 'TFDWConv() only valid for k=4 and p1=1'
+ weight, bias = w.weight.permute(2, 3, 1, 0).numpy(), w.bias.numpy()
+ self.c1 = c1
+ self.conv = [
+ keras.layers.Conv2DTranspose(filters=1,
+ kernel_size=k,
+ strides=s,
+ padding='VALID',
+ output_padding=p2,
+ use_bias=True,
+ kernel_initializer=keras.initializers.Constant(weight[..., i:i + 1]),
+ bias_initializer=keras.initializers.Constant(bias[i])) for i in range(c1)]
+
+ def call(self, inputs):
+ return tf.concat([m(x) for m, x in zip(self.conv, tf.split(inputs, self.c1, 3))], 3)[:, 1:-1, 1:-1]
+
+
+class TFFocus(keras.layers.Layer):
+ # Focus wh information into c-space
+ def __init__(self, c1, c2, k=1, s=1, p=None, g=1, act=True, w=None):
+ # ch_in, ch_out, kernel, stride, padding, groups
+ super().__init__()
+ self.conv = TFConv(c1 * 4, c2, k, s, p, g, act, w.conv)
+
+ def call(self, inputs): # x(b,w,h,c) -> y(b,w/2,h/2,4c)
+ # inputs = inputs / 255 # normalize 0-255 to 0-1
+ inputs = [inputs[:, ::2, ::2, :], inputs[:, 1::2, ::2, :], inputs[:, ::2, 1::2, :], inputs[:, 1::2, 1::2, :]]
+ return self.conv(tf.concat(inputs, 3))
+
+
+class TFBottleneck(keras.layers.Layer):
+ # Standard bottleneck
+ def __init__(self, c1, c2, shortcut=True, g=1, e=0.5, w=None): # ch_in, ch_out, shortcut, groups, expansion
+ super().__init__()
+ c_ = int(c2 * e) # hidden channels
+ self.cv1 = TFConv(c1, c_, 1, 1, w=w.cv1)
+ self.cv2 = TFConv(c_, c2, 3, 1, g=g, w=w.cv2)
+ self.add = shortcut and c1 == c2
+
+ def call(self, inputs):
+ return inputs + self.cv2(self.cv1(inputs)) if self.add else self.cv2(self.cv1(inputs))
+
+
+class TFCrossConv(keras.layers.Layer):
+ # Cross Convolution
+ def __init__(self, c1, c2, k=3, s=1, g=1, e=1.0, shortcut=False, w=None):
+ super().__init__()
+ c_ = int(c2 * e) # hidden channels
+ self.cv1 = TFConv(c1, c_, (1, k), (1, s), w=w.cv1)
+ self.cv2 = TFConv(c_, c2, (k, 1), (s, 1), g=g, w=w.cv2)
+ self.add = shortcut and c1 == c2
+
+ def call(self, inputs):
+ return inputs + self.cv2(self.cv1(inputs)) if self.add else self.cv2(self.cv1(inputs))
+
+
+class TFConv2d(keras.layers.Layer):
+ # Substitution for PyTorch nn.Conv2D
+ def __init__(self, c1, c2, k, s=1, g=1, bias=True, w=None):
+ super().__init__()
+ assert g == 1, "TF v2.2 Conv2D does not support 'groups' argument"
+ self.conv = keras.layers.Conv2D(filters=c2,
+ kernel_size=k,
+ strides=s,
+ padding='VALID',
+ use_bias=bias,
+ kernel_initializer=keras.initializers.Constant(
+ w.weight.permute(2, 3, 1, 0).numpy()),
+ bias_initializer=keras.initializers.Constant(w.bias.numpy()) if bias else None)
+
+ def call(self, inputs):
+ return self.conv(inputs)
+
+
+class TFBottleneckCSP(keras.layers.Layer):
+ # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks
+ def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5, w=None):
+ # ch_in, ch_out, number, shortcut, groups, expansion
+ super().__init__()
+ c_ = int(c2 * e) # hidden channels
+ self.cv1 = TFConv(c1, c_, 1, 1, w=w.cv1)
+ self.cv2 = TFConv2d(c1, c_, 1, 1, bias=False, w=w.cv2)
+ self.cv3 = TFConv2d(c_, c_, 1, 1, bias=False, w=w.cv3)
+ self.cv4 = TFConv(2 * c_, c2, 1, 1, w=w.cv4)
+ self.bn = TFBN(w.bn)
+ self.act = lambda x: keras.activations.swish(x)
+ self.m = keras.Sequential([TFBottleneck(c_, c_, shortcut, g, e=1.0, w=w.m[j]) for j in range(n)])
+
+ def call(self, inputs):
+ y1 = self.cv3(self.m(self.cv1(inputs)))
+ y2 = self.cv2(inputs)
+ return self.cv4(self.act(self.bn(tf.concat((y1, y2), axis=3))))
+
+
+class TFC3(keras.layers.Layer):
+ # CSP Bottleneck with 3 convolutions
+ def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5, w=None):
+ # ch_in, ch_out, number, shortcut, groups, expansion
+ super().__init__()
+ c_ = int(c2 * e) # hidden channels
+ self.cv1 = TFConv(c1, c_, 1, 1, w=w.cv1)
+ self.cv2 = TFConv(c1, c_, 1, 1, w=w.cv2)
+ self.cv3 = TFConv(2 * c_, c2, 1, 1, w=w.cv3)
+ self.m = keras.Sequential([TFBottleneck(c_, c_, shortcut, g, e=1.0, w=w.m[j]) for j in range(n)])
+
+ def call(self, inputs):
+ return self.cv3(tf.concat((self.m(self.cv1(inputs)), self.cv2(inputs)), axis=3))
+
+
+class TFC3x(keras.layers.Layer):
+ # 3 module with cross-convolutions
+ def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5, w=None):
+ # ch_in, ch_out, number, shortcut, groups, expansion
+ super().__init__()
+ c_ = int(c2 * e) # hidden channels
+ self.cv1 = TFConv(c1, c_, 1, 1, w=w.cv1)
+ self.cv2 = TFConv(c1, c_, 1, 1, w=w.cv2)
+ self.cv3 = TFConv(2 * c_, c2, 1, 1, w=w.cv3)
+ self.m = keras.Sequential([
+ TFCrossConv(c_, c_, k=3, s=1, g=g, e=1.0, shortcut=shortcut, w=w.m[j]) for j in range(n)])
+
+ def call(self, inputs):
+ return self.cv3(tf.concat((self.m(self.cv1(inputs)), self.cv2(inputs)), axis=3))
+
+
+class TFSPP(keras.layers.Layer):
+ # Spatial pyramid pooling layer used in YOLOv3-SPP
+ def __init__(self, c1, c2, k=(5, 9, 13), w=None):
+ super().__init__()
+ c_ = c1 // 2 # hidden channels
+ self.cv1 = TFConv(c1, c_, 1, 1, w=w.cv1)
+ self.cv2 = TFConv(c_ * (len(k) + 1), c2, 1, 1, w=w.cv2)
+ self.m = [keras.layers.MaxPool2D(pool_size=x, strides=1, padding='SAME') for x in k]
+
+ def call(self, inputs):
+ x = self.cv1(inputs)
+ return self.cv2(tf.concat([x] + [m(x) for m in self.m], 3))
+
+
+class TFSPPF(keras.layers.Layer):
+ # Spatial pyramid pooling-Fast layer
+ def __init__(self, c1, c2, k=5, w=None):
+ super().__init__()
+ c_ = c1 // 2 # hidden channels
+ self.cv1 = TFConv(c1, c_, 1, 1, w=w.cv1)
+ self.cv2 = TFConv(c_ * 4, c2, 1, 1, w=w.cv2)
+ self.m = keras.layers.MaxPool2D(pool_size=k, strides=1, padding='SAME')
+
+ def call(self, inputs):
+ x = self.cv1(inputs)
+ y1 = self.m(x)
+ y2 = self.m(y1)
+ return self.cv2(tf.concat([x, y1, y2, self.m(y2)], 3))
+
+
+class TFDetect(keras.layers.Layer):
+ # TF YOLO Detect layer
+ def __init__(self, nc=80, anchors=(), ch=(), imgsz=(640, 640), w=None): # detection layer
+ super().__init__()
+ self.stride = tf.convert_to_tensor(w.stride.numpy(), dtype=tf.float32)
+ self.nc = nc # number of classes
+ self.no = nc + 5 # number of outputs per anchor
+ self.nl = len(anchors) # number of detection layers
+ self.na = len(anchors[0]) // 2 # number of anchors
+ self.grid = [tf.zeros(1)] * self.nl # init grid
+ self.anchors = tf.convert_to_tensor(w.anchors.numpy(), dtype=tf.float32)
+ self.anchor_grid = tf.reshape(self.anchors * tf.reshape(self.stride, [self.nl, 1, 1]), [self.nl, 1, -1, 1, 2])
+ self.m = [TFConv2d(x, self.no * self.na, 1, w=w.m[i]) for i, x in enumerate(ch)]
+ self.training = False # set to False after building model
+ self.imgsz = imgsz
+ for i in range(self.nl):
+ ny, nx = self.imgsz[0] // self.stride[i], self.imgsz[1] // self.stride[i]
+ self.grid[i] = self._make_grid(nx, ny)
+
+ def call(self, inputs):
+ z = [] # inference output
+ x = []
+ for i in range(self.nl):
+ x.append(self.m[i](inputs[i]))
+ # x(bs,20,20,255) to x(bs,3,20,20,85)
+ ny, nx = self.imgsz[0] // self.stride[i], self.imgsz[1] // self.stride[i]
+ x[i] = tf.reshape(x[i], [-1, ny * nx, self.na, self.no])
+
+ if not self.training: # inference
+ y = x[i]
+ grid = tf.transpose(self.grid[i], [0, 2, 1, 3]) - 0.5
+ anchor_grid = tf.transpose(self.anchor_grid[i], [0, 2, 1, 3]) * 4
+ xy = (tf.sigmoid(y[..., 0:2]) * 2 + grid) * self.stride[i] # xy
+ wh = tf.sigmoid(y[..., 2:4]) ** 2 * anchor_grid
+ # Normalize xywh to 0-1 to reduce calibration error
+ xy /= tf.constant([[self.imgsz[1], self.imgsz[0]]], dtype=tf.float32)
+ wh /= tf.constant([[self.imgsz[1], self.imgsz[0]]], dtype=tf.float32)
+ y = tf.concat([xy, wh, tf.sigmoid(y[..., 4:5 + self.nc]), y[..., 5 + self.nc:]], -1)
+ z.append(tf.reshape(y, [-1, self.na * ny * nx, self.no]))
+
+ return tf.transpose(x, [0, 2, 1, 3]) if self.training else (tf.concat(z, 1),)
+
+ @staticmethod
+ def _make_grid(nx=20, ny=20):
+ # yv, xv = torch.meshgrid([torch.arange(ny), torch.arange(nx)])
+ # return torch.stack((xv, yv), 2).view((1, 1, ny, nx, 2)).float()
+ xv, yv = tf.meshgrid(tf.range(nx), tf.range(ny))
+ return tf.cast(tf.reshape(tf.stack([xv, yv], 2), [1, 1, ny * nx, 2]), dtype=tf.float32)
+
+
+class TFSegment(TFDetect):
+ # YOLO Segment head for segmentation models
+ def __init__(self, nc=80, anchors=(), nm=32, npr=256, ch=(), imgsz=(640, 640), w=None):
+ super().__init__(nc, anchors, ch, imgsz, w)
+ self.nm = nm # number of masks
+ self.npr = npr # number of protos
+ self.no = 5 + nc + self.nm # number of outputs per anchor
+ self.m = [TFConv2d(x, self.no * self.na, 1, w=w.m[i]) for i, x in enumerate(ch)] # output conv
+ self.proto = TFProto(ch[0], self.npr, self.nm, w=w.proto) # protos
+ self.detect = TFDetect.call
+
+ def call(self, x):
+ p = self.proto(x[0])
+ # p = TFUpsample(None, scale_factor=4, mode='nearest')(self.proto(x[0])) # (optional) full-size protos
+ p = tf.transpose(p, [0, 3, 1, 2]) # from shape(1,160,160,32) to shape(1,32,160,160)
+ x = self.detect(self, x)
+ return (x, p) if self.training else (x[0], p)
+
+
+class TFProto(keras.layers.Layer):
+
+ def __init__(self, c1, c_=256, c2=32, w=None):
+ super().__init__()
+ self.cv1 = TFConv(c1, c_, k=3, w=w.cv1)
+ self.upsample = TFUpsample(None, scale_factor=2, mode='nearest')
+ self.cv2 = TFConv(c_, c_, k=3, w=w.cv2)
+ self.cv3 = TFConv(c_, c2, w=w.cv3)
+
+ def call(self, inputs):
+ return self.cv3(self.cv2(self.upsample(self.cv1(inputs))))
+
+
+class TFUpsample(keras.layers.Layer):
+ # TF version of torch.nn.Upsample()
+ def __init__(self, size, scale_factor, mode, w=None): # warning: all arguments needed including 'w'
+ super().__init__()
+ assert scale_factor % 2 == 0, "scale_factor must be multiple of 2"
+ self.upsample = lambda x: tf.image.resize(x, (x.shape[1] * scale_factor, x.shape[2] * scale_factor), mode)
+ # self.upsample = keras.layers.UpSampling2D(size=scale_factor, interpolation=mode)
+ # with default arguments: align_corners=False, half_pixel_centers=False
+ # self.upsample = lambda x: tf.raw_ops.ResizeNearestNeighbor(images=x,
+ # size=(x.shape[1] * 2, x.shape[2] * 2))
+
+ def call(self, inputs):
+ return self.upsample(inputs)
+
+
+class TFConcat(keras.layers.Layer):
+ # TF version of torch.concat()
+ def __init__(self, dimension=1, w=None):
+ super().__init__()
+ assert dimension == 1, "convert only NCHW to NHWC concat"
+ self.d = 3
+
+ def call(self, inputs):
+ return tf.concat(inputs, self.d)
+
+
+def parse_model(d, ch, model, imgsz): # model_dict, input_channels(3)
+ LOGGER.info(f"\n{'':>3}{'from':>18}{'n':>3}{'params':>10} {'module':<40}{'arguments':<30}")
+ anchors, nc, gd, gw = d['anchors'], d['nc'], d['depth_multiple'], d['width_multiple']
+ na = (len(anchors[0]) // 2) if isinstance(anchors, list) else anchors # number of anchors
+ no = na * (nc + 5) # number of outputs = anchors * (classes + 5)
+
+ layers, save, c2 = [], [], ch[-1] # layers, savelist, ch out
+ for i, (f, n, m, args) in enumerate(d['backbone'] + d['head']): # from, number, module, args
+ m_str = m
+ m = eval(m) if isinstance(m, str) else m # eval strings
+ for j, a in enumerate(args):
+ try:
+ args[j] = eval(a) if isinstance(a, str) else a # eval strings
+ except NameError:
+ pass
+
+ n = max(round(n * gd), 1) if n > 1 else n # depth gain
+ if m in [
+ nn.Conv2d, Conv, DWConv, DWConvTranspose2d, Bottleneck, SPP, SPPF, MixConv2d, Focus, CrossConv,
+ BottleneckCSP, C3, C3x]:
+ c1, c2 = ch[f], args[0]
+ c2 = make_divisible(c2 * gw, 8) if c2 != no else c2
+
+ args = [c1, c2, *args[1:]]
+ if m in [BottleneckCSP, C3, C3x]:
+ args.insert(2, n)
+ n = 1
+ elif m is nn.BatchNorm2d:
+ args = [ch[f]]
+ elif m is Concat:
+ c2 = sum(ch[-1 if x == -1 else x + 1] for x in f)
+ elif m in [Detect, Segment]:
+ args.append([ch[x + 1] for x in f])
+ if isinstance(args[1], int): # number of anchors
+ args[1] = [list(range(args[1] * 2))] * len(f)
+ if m is Segment:
+ args[3] = make_divisible(args[3] * gw, 8)
+ args.append(imgsz)
+ else:
+ c2 = ch[f]
+
+ tf_m = eval('TF' + m_str.replace('nn.', ''))
+ m_ = keras.Sequential([tf_m(*args, w=model.model[i][j]) for j in range(n)]) if n > 1 \
+ else tf_m(*args, w=model.model[i]) # module
+
+ torch_m_ = nn.Sequential(*(m(*args) for _ in range(n))) if n > 1 else m(*args) # module
+ t = str(m)[8:-2].replace('__main__.', '') # module type
+ np = sum(x.numel() for x in torch_m_.parameters()) # number params
+ m_.i, m_.f, m_.type, m_.np = i, f, t, np # attach index, 'from' index, type, number params
+ LOGGER.info(f'{i:>3}{str(f):>18}{str(n):>3}{np:>10} {t:<40}{str(args):<30}') # print
+ save.extend(x % i for x in ([f] if isinstance(f, int) else f) if x != -1) # append to savelist
+ layers.append(m_)
+ ch.append(c2)
+ return keras.Sequential(layers), sorted(save)
+
+
+class TFModel:
+ # TF YOLO model
+ def __init__(self, cfg='yolo.yaml', ch=3, nc=None, model=None, imgsz=(640, 640)): # model, channels, classes
+ super().__init__()
+ if isinstance(cfg, dict):
+ self.yaml = cfg # model dict
+ else: # is *.yaml
+ import yaml # for torch hub
+ self.yaml_file = Path(cfg).name
+ with open(cfg) as f:
+ self.yaml = yaml.load(f, Loader=yaml.FullLoader) # model dict
+
+ # Define model
+ if nc and nc != self.yaml['nc']:
+ LOGGER.info(f"Overriding {cfg} nc={self.yaml['nc']} with nc={nc}")
+ self.yaml['nc'] = nc # override yaml value
+ self.model, self.savelist = parse_model(deepcopy(self.yaml), ch=[ch], model=model, imgsz=imgsz)
+
+ def predict(self,
+ inputs,
+ tf_nms=False,
+ agnostic_nms=False,
+ topk_per_class=100,
+ topk_all=100,
+ iou_thres=0.45,
+ conf_thres=0.25):
+ y = [] # outputs
+ x = inputs
+ for m in self.model.layers:
+ if m.f != -1: # if not from previous layer
+ x = y[m.f] if isinstance(m.f, int) else [x if j == -1 else y[j] for j in m.f] # from earlier layers
+
+ x = m(x) # run
+ y.append(x if m.i in self.savelist else None) # save output
+
+ # Add TensorFlow NMS
+ if tf_nms:
+ boxes = self._xywh2xyxy(x[0][..., :4])
+ probs = x[0][:, :, 4:5]
+ classes = x[0][:, :, 5:]
+ scores = probs * classes
+ if agnostic_nms:
+ nms = AgnosticNMS()((boxes, classes, scores), topk_all, iou_thres, conf_thres)
+ else:
+ boxes = tf.expand_dims(boxes, 2)
+ nms = tf.image.combined_non_max_suppression(boxes,
+ scores,
+ topk_per_class,
+ topk_all,
+ iou_thres,
+ conf_thres,
+ clip_boxes=False)
+ return (nms,)
+ return x # output [1,6300,85] = [xywh, conf, class0, class1, ...]
+ # x = x[0] # [x(1,6300,85), ...] to x(6300,85)
+ # xywh = x[..., :4] # x(6300,4) boxes
+ # conf = x[..., 4:5] # x(6300,1) confidences
+ # cls = tf.reshape(tf.cast(tf.argmax(x[..., 5:], axis=1), tf.float32), (-1, 1)) # x(6300,1) classes
+ # return tf.concat([conf, cls, xywh], 1)
+
+ @staticmethod
+ def _xywh2xyxy(xywh):
+ # Convert nx4 boxes from [x, y, w, h] to [x1, y1, x2, y2] where xy1=top-left, xy2=bottom-right
+ x, y, w, h = tf.split(xywh, num_or_size_splits=4, axis=-1)
+ return tf.concat([x - w / 2, y - h / 2, x + w / 2, y + h / 2], axis=-1)
+
+
+class AgnosticNMS(keras.layers.Layer):
+ # TF Agnostic NMS
+ def call(self, input, topk_all, iou_thres, conf_thres):
+ # wrap map_fn to avoid TypeSpec related error https://stackoverflow.com/a/65809989/3036450
+ return tf.map_fn(lambda x: self._nms(x, topk_all, iou_thres, conf_thres),
+ input,
+ fn_output_signature=(tf.float32, tf.float32, tf.float32, tf.int32),
+ name='agnostic_nms')
+
+ @staticmethod
+ def _nms(x, topk_all=100, iou_thres=0.45, conf_thres=0.25): # agnostic NMS
+ boxes, classes, scores = x
+ class_inds = tf.cast(tf.argmax(classes, axis=-1), tf.float32)
+ scores_inp = tf.reduce_max(scores, -1)
+ selected_inds = tf.image.non_max_suppression(boxes,
+ scores_inp,
+ max_output_size=topk_all,
+ iou_threshold=iou_thres,
+ score_threshold=conf_thres)
+ selected_boxes = tf.gather(boxes, selected_inds)
+ padded_boxes = tf.pad(selected_boxes,
+ paddings=[[0, topk_all - tf.shape(selected_boxes)[0]], [0, 0]],
+ mode="CONSTANT",
+ constant_values=0.0)
+ selected_scores = tf.gather(scores_inp, selected_inds)
+ padded_scores = tf.pad(selected_scores,
+ paddings=[[0, topk_all - tf.shape(selected_boxes)[0]]],
+ mode="CONSTANT",
+ constant_values=-1.0)
+ selected_classes = tf.gather(class_inds, selected_inds)
+ padded_classes = tf.pad(selected_classes,
+ paddings=[[0, topk_all - tf.shape(selected_boxes)[0]]],
+ mode="CONSTANT",
+ constant_values=-1.0)
+ valid_detections = tf.shape(selected_inds)[0]
+ return padded_boxes, padded_scores, padded_classes, valid_detections
+
+
+def activations(act=nn.SiLU):
+ # Returns TF activation from input PyTorch activation
+ if isinstance(act, nn.LeakyReLU):
+ return lambda x: keras.activations.relu(x, alpha=0.1)
+ elif isinstance(act, nn.Hardswish):
+ return lambda x: x * tf.nn.relu6(x + 3) * 0.166666667
+ elif isinstance(act, (nn.SiLU, SiLU)):
+ return lambda x: keras.activations.swish(x)
+ else:
+ raise Exception(f'no matching TensorFlow activation found for PyTorch activation {act}')
+
+
+def representative_dataset_gen(dataset, ncalib=100):
+ # Representative dataset generator for use with converter.representative_dataset, returns a generator of np arrays
+ for n, (path, img, im0s, vid_cap, string) in enumerate(dataset):
+ im = np.transpose(img, [1, 2, 0])
+ im = np.expand_dims(im, axis=0).astype(np.float32)
+ im /= 255
+ yield [im]
+ if n >= ncalib:
+ break
+
+
+def run(
+ weights=ROOT / 'yolo.pt', # weights path
+ imgsz=(640, 640), # inference size h,w
+ batch_size=1, # batch size
+ dynamic=False, # dynamic batch size
+):
+ # PyTorch model
+ im = torch.zeros((batch_size, 3, *imgsz)) # BCHW image
+ model = attempt_load(weights, device=torch.device('cpu'), inplace=True, fuse=False)
+ _ = model(im) # inference
+ model.info()
+
+ # TensorFlow model
+ im = tf.zeros((batch_size, *imgsz, 3)) # BHWC image
+ tf_model = TFModel(cfg=model.yaml, model=model, nc=model.nc, imgsz=imgsz)
+ _ = tf_model.predict(im) # inference
+
+ # Keras model
+ im = keras.Input(shape=(*imgsz, 3), batch_size=None if dynamic else batch_size)
+ keras_model = keras.Model(inputs=im, outputs=tf_model.predict(im))
+ keras_model.summary()
+
+ LOGGER.info('PyTorch, TensorFlow and Keras models successfully verified.\nUse export.py for TF model export.')
+
+
+def parse_opt():
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--weights', type=str, default=ROOT / 'yolo.pt', help='weights path')
+ parser.add_argument('--imgsz', '--img', '--img-size', nargs='+', type=int, default=[640], help='inference size h,w')
+ parser.add_argument('--batch-size', type=int, default=1, help='batch size')
+ parser.add_argument('--dynamic', action='store_true', help='dynamic batch size')
+ opt = parser.parse_args()
+ opt.imgsz *= 2 if len(opt.imgsz) == 1 else 1 # expand
+ print_args(vars(opt))
+ return opt
+
+
+def main(opt):
+ run(**vars(opt))
+
+
+if __name__ == "__main__":
+ opt = parse_opt()
+ main(opt)
diff --git a/models/yolo.py b/models/yolo.py
new file mode 100644
index 0000000000000000000000000000000000000000..a28c7d30562d1e683ddd6c7f41d905681091a182
--- /dev/null
+++ b/models/yolo.py
@@ -0,0 +1,818 @@
+import argparse
+import os
+import platform
+import sys
+from copy import deepcopy
+from pathlib import Path
+
+FILE = Path(__file__).resolve()
+ROOT = FILE.parents[1] # YOLO root directory
+if str(ROOT) not in sys.path:
+ sys.path.append(str(ROOT)) # add ROOT to PATH
+if platform.system() != 'Windows':
+ ROOT = Path(os.path.relpath(ROOT, Path.cwd())) # relative
+
+from models.common import *
+from models.experimental import *
+from utils.general import LOGGER, check_version, check_yaml, make_divisible, print_args
+from utils.plots import feature_visualization
+from utils.torch_utils import (fuse_conv_and_bn, initialize_weights, model_info, profile, scale_img, select_device,
+ time_sync)
+from utils.tal.anchor_generator import make_anchors, dist2bbox
+
+try:
+ import thop # for FLOPs computation
+except ImportError:
+ thop = None
+
+
+class Detect(nn.Module):
+ # YOLO Detect head for detection models
+ dynamic = False # force grid reconstruction
+ export = False # export mode
+ shape = None
+ anchors = torch.empty(0) # init
+ strides = torch.empty(0) # init
+
+ def __init__(self, nc=80, ch=(), inplace=True): # detection layer
+ super().__init__()
+ self.nc = nc # number of classes
+ self.nl = len(ch) # number of detection layers
+ self.reg_max = 16
+ self.no = nc + self.reg_max * 4 # number of outputs per anchor
+ self.inplace = inplace # use inplace ops (e.g. slice assignment)
+ self.stride = torch.zeros(self.nl) # strides computed during build
+
+ c2, c3 = max((ch[0] // 4, self.reg_max * 4, 16)), max((ch[0], min((self.nc * 2, 128)))) # channels
+ self.cv2 = nn.ModuleList(
+ nn.Sequential(Conv(x, c2, 3), Conv(c2, c2, 3), nn.Conv2d(c2, 4 * self.reg_max, 1)) for x in ch)
+ self.cv3 = nn.ModuleList(
+ nn.Sequential(Conv(x, c3, 3), Conv(c3, c3, 3), nn.Conv2d(c3, self.nc, 1)) for x in ch)
+ self.dfl = DFL(self.reg_max) if self.reg_max > 1 else nn.Identity()
+
+ def forward(self, x):
+ shape = x[0].shape # BCHW
+ for i in range(self.nl):
+ x[i] = torch.cat((self.cv2[i](x[i]), self.cv3[i](x[i])), 1)
+ if self.training:
+ return x
+ elif self.dynamic or self.shape != shape:
+ self.anchors, self.strides = (x.transpose(0, 1) for x in make_anchors(x, self.stride, 0.5))
+ self.shape = shape
+
+ box, cls = torch.cat([xi.view(shape[0], self.no, -1) for xi in x], 2).split((self.reg_max * 4, self.nc), 1)
+ dbox = dist2bbox(self.dfl(box), self.anchors.unsqueeze(0), xywh=True, dim=1) * self.strides
+ y = torch.cat((dbox, cls.sigmoid()), 1)
+ return y if self.export else (y, x)
+
+ def bias_init(self):
+ # Initialize Detect() biases, WARNING: requires stride availability
+ m = self # self.model[-1] # Detect() module
+ # cf = torch.bincount(torch.tensor(np.concatenate(dataset.labels, 0)[:, 0]).long(), minlength=nc) + 1
+ # ncf = math.log(0.6 / (m.nc - 0.999999)) if cf is None else torch.log(cf / cf.sum()) # nominal class frequency
+ for a, b, s in zip(m.cv2, m.cv3, m.stride): # from
+ a[-1].bias.data[:] = 1.0 # box
+ b[-1].bias.data[:m.nc] = math.log(5 / m.nc / (640 / s) ** 2) # cls (5 objects and 80 classes per 640 image)
+
+
+class DDetect(nn.Module):
+ # YOLO Detect head for detection models
+ dynamic = False # force grid reconstruction
+ export = False # export mode
+ shape = None
+ anchors = torch.empty(0) # init
+ strides = torch.empty(0) # init
+
+ def __init__(self, nc=80, ch=(), inplace=True): # detection layer
+ super().__init__()
+ self.nc = nc # number of classes
+ self.nl = len(ch) # number of detection layers
+ self.reg_max = 16
+ self.no = nc + self.reg_max * 4 # number of outputs per anchor
+ self.inplace = inplace # use inplace ops (e.g. slice assignment)
+ self.stride = torch.zeros(self.nl) # strides computed during build
+
+ c2, c3 = make_divisible(max((ch[0] // 4, self.reg_max * 4, 16)), 4), max((ch[0], min((self.nc * 2, 128)))) # channels
+ self.cv2 = nn.ModuleList(
+ nn.Sequential(Conv(x, c2, 3), Conv(c2, c2, 3, g=4), nn.Conv2d(c2, 4 * self.reg_max, 1, groups=4)) for x in ch)
+ self.cv3 = nn.ModuleList(
+ nn.Sequential(Conv(x, c3, 3), Conv(c3, c3, 3), nn.Conv2d(c3, self.nc, 1)) for x in ch)
+ self.dfl = DFL(self.reg_max) if self.reg_max > 1 else nn.Identity()
+
+ def forward(self, x):
+ shape = x[0].shape # BCHW
+ for i in range(self.nl):
+ x[i] = torch.cat((self.cv2[i](x[i]), self.cv3[i](x[i])), 1)
+ if self.training:
+ return x
+ elif self.dynamic or self.shape != shape:
+ self.anchors, self.strides = (x.transpose(0, 1) for x in make_anchors(x, self.stride, 0.5))
+ self.shape = shape
+
+ box, cls = torch.cat([xi.view(shape[0], self.no, -1) for xi in x], 2).split((self.reg_max * 4, self.nc), 1)
+ dbox = dist2bbox(self.dfl(box), self.anchors.unsqueeze(0), xywh=True, dim=1) * self.strides
+ y = torch.cat((dbox, cls.sigmoid()), 1)
+ return y if self.export else (y, x)
+
+ def bias_init(self):
+ # Initialize Detect() biases, WARNING: requires stride availability
+ m = self # self.model[-1] # Detect() module
+ # cf = torch.bincount(torch.tensor(np.concatenate(dataset.labels, 0)[:, 0]).long(), minlength=nc) + 1
+ # ncf = math.log(0.6 / (m.nc - 0.999999)) if cf is None else torch.log(cf / cf.sum()) # nominal class frequency
+ for a, b, s in zip(m.cv2, m.cv3, m.stride): # from
+ a[-1].bias.data[:] = 1.0 # box
+ b[-1].bias.data[:m.nc] = math.log(5 / m.nc / (640 / s) ** 2) # cls (5 objects and 80 classes per 640 image)
+
+
+class DualDetect(nn.Module):
+ # YOLO Detect head for detection models
+ dynamic = False # force grid reconstruction
+ export = False # export mode
+ shape = None
+ anchors = torch.empty(0) # init
+ strides = torch.empty(0) # init
+
+ def __init__(self, nc=80, ch=(), inplace=True): # detection layer
+ super().__init__()
+ self.nc = nc # number of classes
+ self.nl = len(ch) // 2 # number of detection layers
+ self.reg_max = 16
+ self.no = nc + self.reg_max * 4 # number of outputs per anchor
+ self.inplace = inplace # use inplace ops (e.g. slice assignment)
+ self.stride = torch.zeros(self.nl) # strides computed during build
+
+ c2, c3 = max((ch[0] // 4, self.reg_max * 4, 16)), max((ch[0], min((self.nc * 2, 128)))) # channels
+ c4, c5 = max((ch[self.nl] // 4, self.reg_max * 4, 16)), max((ch[self.nl], min((self.nc * 2, 128)))) # channels
+ self.cv2 = nn.ModuleList(
+ nn.Sequential(Conv(x, c2, 3), Conv(c2, c2, 3), nn.Conv2d(c2, 4 * self.reg_max, 1)) for x in ch[:self.nl])
+ self.cv3 = nn.ModuleList(
+ nn.Sequential(Conv(x, c3, 3), Conv(c3, c3, 3), nn.Conv2d(c3, self.nc, 1)) for x in ch[:self.nl])
+ self.cv4 = nn.ModuleList(
+ nn.Sequential(Conv(x, c4, 3), Conv(c4, c4, 3), nn.Conv2d(c4, 4 * self.reg_max, 1)) for x in ch[self.nl:])
+ self.cv5 = nn.ModuleList(
+ nn.Sequential(Conv(x, c5, 3), Conv(c5, c5, 3), nn.Conv2d(c5, self.nc, 1)) for x in ch[self.nl:])
+ self.dfl = DFL(self.reg_max)
+ self.dfl2 = DFL(self.reg_max)
+
+ def forward(self, x):
+ shape = x[0].shape # BCHW
+ d1 = []
+ d2 = []
+ for i in range(self.nl):
+ d1.append(torch.cat((self.cv2[i](x[i]), self.cv3[i](x[i])), 1))
+ d2.append(torch.cat((self.cv4[i](x[self.nl+i]), self.cv5[i](x[self.nl+i])), 1))
+ if self.training:
+ return [d1, d2]
+ elif self.dynamic or self.shape != shape:
+ self.anchors, self.strides = (d1.transpose(0, 1) for d1 in make_anchors(d1, self.stride, 0.5))
+ self.shape = shape
+
+ box, cls = torch.cat([di.view(shape[0], self.no, -1) for di in d1], 2).split((self.reg_max * 4, self.nc), 1)
+ dbox = dist2bbox(self.dfl(box), self.anchors.unsqueeze(0), xywh=True, dim=1) * self.strides
+ box2, cls2 = torch.cat([di.view(shape[0], self.no, -1) for di in d2], 2).split((self.reg_max * 4, self.nc), 1)
+ dbox2 = dist2bbox(self.dfl2(box2), self.anchors.unsqueeze(0), xywh=True, dim=1) * self.strides
+ y = [torch.cat((dbox, cls.sigmoid()), 1), torch.cat((dbox2, cls2.sigmoid()), 1)]
+ return y if self.export else (y, [d1, d2])
+
+ def bias_init(self):
+ # Initialize Detect() biases, WARNING: requires stride availability
+ m = self # self.model[-1] # Detect() module
+ # cf = torch.bincount(torch.tensor(np.concatenate(dataset.labels, 0)[:, 0]).long(), minlength=nc) + 1
+ # ncf = math.log(0.6 / (m.nc - 0.999999)) if cf is None else torch.log(cf / cf.sum()) # nominal class frequency
+ for a, b, s in zip(m.cv2, m.cv3, m.stride): # from
+ a[-1].bias.data[:] = 1.0 # box
+ b[-1].bias.data[:m.nc] = math.log(5 / m.nc / (640 / s) ** 2) # cls (5 objects and 80 classes per 640 image)
+ for a, b, s in zip(m.cv4, m.cv5, m.stride): # from
+ a[-1].bias.data[:] = 1.0 # box
+ b[-1].bias.data[:m.nc] = math.log(5 / m.nc / (640 / s) ** 2) # cls (5 objects and 80 classes per 640 image)
+
+
+class DualDDetect(nn.Module):
+ # YOLO Detect head for detection models
+ dynamic = False # force grid reconstruction
+ export = False # export mode
+ shape = None
+ anchors = torch.empty(0) # init
+ strides = torch.empty(0) # init
+
+ def __init__(self, nc=80, ch=(), inplace=True): # detection layer
+ super().__init__()
+ self.nc = nc # number of classes
+ self.nl = len(ch) // 2 # number of detection layers
+ self.reg_max = 16
+ self.no = nc + self.reg_max * 4 # number of outputs per anchor
+ self.inplace = inplace # use inplace ops (e.g. slice assignment)
+ self.stride = torch.zeros(self.nl) # strides computed during build
+
+ c2, c3 = make_divisible(max((ch[0] // 4, self.reg_max * 4, 16)), 4), max((ch[0], min((self.nc * 2, 128)))) # channels
+ c4, c5 = make_divisible(max((ch[self.nl] // 4, self.reg_max * 4, 16)), 4), max((ch[self.nl], min((self.nc * 2, 128)))) # channels
+ self.cv2 = nn.ModuleList(
+ nn.Sequential(Conv(x, c2, 3), Conv(c2, c2, 3, g=4), nn.Conv2d(c2, 4 * self.reg_max, 1, groups=4)) for x in ch[:self.nl])
+ self.cv3 = nn.ModuleList(
+ nn.Sequential(Conv(x, c3, 3), Conv(c3, c3, 3), nn.Conv2d(c3, self.nc, 1)) for x in ch[:self.nl])
+ self.cv4 = nn.ModuleList(
+ nn.Sequential(Conv(x, c4, 3), Conv(c4, c4, 3, g=4), nn.Conv2d(c4, 4 * self.reg_max, 1, groups=4)) for x in ch[self.nl:])
+ self.cv5 = nn.ModuleList(
+ nn.Sequential(Conv(x, c5, 3), Conv(c5, c5, 3), nn.Conv2d(c5, self.nc, 1)) for x in ch[self.nl:])
+ self.dfl = DFL(self.reg_max)
+ self.dfl2 = DFL(self.reg_max)
+
+ def forward(self, x):
+ shape = x[0].shape # BCHW
+ d1 = []
+ d2 = []
+ for i in range(self.nl):
+ d1.append(torch.cat((self.cv2[i](x[i]), self.cv3[i](x[i])), 1))
+ d2.append(torch.cat((self.cv4[i](x[self.nl+i]), self.cv5[i](x[self.nl+i])), 1))
+ if self.training:
+ return [d1, d2]
+ elif self.dynamic or self.shape != shape:
+ self.anchors, self.strides = (d1.transpose(0, 1) for d1 in make_anchors(d1, self.stride, 0.5))
+ self.shape = shape
+
+ box, cls = torch.cat([di.view(shape[0], self.no, -1) for di in d1], 2).split((self.reg_max * 4, self.nc), 1)
+ dbox = dist2bbox(self.dfl(box), self.anchors.unsqueeze(0), xywh=True, dim=1) * self.strides
+ box2, cls2 = torch.cat([di.view(shape[0], self.no, -1) for di in d2], 2).split((self.reg_max * 4, self.nc), 1)
+ dbox2 = dist2bbox(self.dfl2(box2), self.anchors.unsqueeze(0), xywh=True, dim=1) * self.strides
+ y = [torch.cat((dbox, cls.sigmoid()), 1), torch.cat((dbox2, cls2.sigmoid()), 1)]
+ return y if self.export else (y, [d1, d2])
+ #y = torch.cat((dbox2, cls2.sigmoid()), 1)
+ #return y if self.export else (y, d2)
+ #y1 = torch.cat((dbox, cls.sigmoid()), 1)
+ #y2 = torch.cat((dbox2, cls2.sigmoid()), 1)
+ #return [y1, y2] if self.export else [(y1, d1), (y2, d2)]
+ #return [y1, y2] if self.export else [(y1, y2), (d1, d2)]
+
+ def bias_init(self):
+ # Initialize Detect() biases, WARNING: requires stride availability
+ m = self # self.model[-1] # Detect() module
+ # cf = torch.bincount(torch.tensor(np.concatenate(dataset.labels, 0)[:, 0]).long(), minlength=nc) + 1
+ # ncf = math.log(0.6 / (m.nc - 0.999999)) if cf is None else torch.log(cf / cf.sum()) # nominal class frequency
+ for a, b, s in zip(m.cv2, m.cv3, m.stride): # from
+ a[-1].bias.data[:] = 1.0 # box
+ b[-1].bias.data[:m.nc] = math.log(5 / m.nc / (640 / s) ** 2) # cls (5 objects and 80 classes per 640 image)
+ for a, b, s in zip(m.cv4, m.cv5, m.stride): # from
+ a[-1].bias.data[:] = 1.0 # box
+ b[-1].bias.data[:m.nc] = math.log(5 / m.nc / (640 / s) ** 2) # cls (5 objects and 80 classes per 640 image)
+
+
+class TripleDetect(nn.Module):
+ # YOLO Detect head for detection models
+ dynamic = False # force grid reconstruction
+ export = False # export mode
+ shape = None
+ anchors = torch.empty(0) # init
+ strides = torch.empty(0) # init
+
+ def __init__(self, nc=80, ch=(), inplace=True): # detection layer
+ super().__init__()
+ self.nc = nc # number of classes
+ self.nl = len(ch) // 3 # number of detection layers
+ self.reg_max = 16
+ self.no = nc + self.reg_max * 4 # number of outputs per anchor
+ self.inplace = inplace # use inplace ops (e.g. slice assignment)
+ self.stride = torch.zeros(self.nl) # strides computed during build
+
+ c2, c3 = max((ch[0] // 4, self.reg_max * 4, 16)), max((ch[0], min((self.nc * 2, 128)))) # channels
+ c4, c5 = max((ch[self.nl] // 4, self.reg_max * 4, 16)), max((ch[self.nl], min((self.nc * 2, 128)))) # channels
+ c6, c7 = max((ch[self.nl * 2] // 4, self.reg_max * 4, 16)), max((ch[self.nl * 2], min((self.nc * 2, 128)))) # channels
+ self.cv2 = nn.ModuleList(
+ nn.Sequential(Conv(x, c2, 3), Conv(c2, c2, 3), nn.Conv2d(c2, 4 * self.reg_max, 1)) for x in ch[:self.nl])
+ self.cv3 = nn.ModuleList(
+ nn.Sequential(Conv(x, c3, 3), Conv(c3, c3, 3), nn.Conv2d(c3, self.nc, 1)) for x in ch[:self.nl])
+ self.cv4 = nn.ModuleList(
+ nn.Sequential(Conv(x, c4, 3), Conv(c4, c4, 3), nn.Conv2d(c4, 4 * self.reg_max, 1)) for x in ch[self.nl:self.nl*2])
+ self.cv5 = nn.ModuleList(
+ nn.Sequential(Conv(x, c5, 3), Conv(c5, c5, 3), nn.Conv2d(c5, self.nc, 1)) for x in ch[self.nl:self.nl*2])
+ self.cv6 = nn.ModuleList(
+ nn.Sequential(Conv(x, c6, 3), Conv(c6, c6, 3), nn.Conv2d(c6, 4 * self.reg_max, 1)) for x in ch[self.nl*2:self.nl*3])
+ self.cv7 = nn.ModuleList(
+ nn.Sequential(Conv(x, c7, 3), Conv(c7, c7, 3), nn.Conv2d(c7, self.nc, 1)) for x in ch[self.nl*2:self.nl*3])
+ self.dfl = DFL(self.reg_max)
+ self.dfl2 = DFL(self.reg_max)
+ self.dfl3 = DFL(self.reg_max)
+
+ def forward(self, x):
+ shape = x[0].shape # BCHW
+ d1 = []
+ d2 = []
+ d3 = []
+ for i in range(self.nl):
+ d1.append(torch.cat((self.cv2[i](x[i]), self.cv3[i](x[i])), 1))
+ d2.append(torch.cat((self.cv4[i](x[self.nl+i]), self.cv5[i](x[self.nl+i])), 1))
+ d3.append(torch.cat((self.cv6[i](x[self.nl*2+i]), self.cv7[i](x[self.nl*2+i])), 1))
+ if self.training:
+ return [d1, d2, d3]
+ elif self.dynamic or self.shape != shape:
+ self.anchors, self.strides = (d1.transpose(0, 1) for d1 in make_anchors(d1, self.stride, 0.5))
+ self.shape = shape
+
+ box, cls = torch.cat([di.view(shape[0], self.no, -1) for di in d1], 2).split((self.reg_max * 4, self.nc), 1)
+ dbox = dist2bbox(self.dfl(box), self.anchors.unsqueeze(0), xywh=True, dim=1) * self.strides
+ box2, cls2 = torch.cat([di.view(shape[0], self.no, -1) for di in d2], 2).split((self.reg_max * 4, self.nc), 1)
+ dbox2 = dist2bbox(self.dfl2(box2), self.anchors.unsqueeze(0), xywh=True, dim=1) * self.strides
+ box3, cls3 = torch.cat([di.view(shape[0], self.no, -1) for di in d3], 2).split((self.reg_max * 4, self.nc), 1)
+ dbox3 = dist2bbox(self.dfl3(box3), self.anchors.unsqueeze(0), xywh=True, dim=1) * self.strides
+ y = [torch.cat((dbox, cls.sigmoid()), 1), torch.cat((dbox2, cls2.sigmoid()), 1), torch.cat((dbox3, cls3.sigmoid()), 1)]
+ return y if self.export else (y, [d1, d2, d3])
+
+ def bias_init(self):
+ # Initialize Detect() biases, WARNING: requires stride availability
+ m = self # self.model[-1] # Detect() module
+ # cf = torch.bincount(torch.tensor(np.concatenate(dataset.labels, 0)[:, 0]).long(), minlength=nc) + 1
+ # ncf = math.log(0.6 / (m.nc - 0.999999)) if cf is None else torch.log(cf / cf.sum()) # nominal class frequency
+ for a, b, s in zip(m.cv2, m.cv3, m.stride): # from
+ a[-1].bias.data[:] = 1.0 # box
+ b[-1].bias.data[:m.nc] = math.log(5 / m.nc / (640 / s) ** 2) # cls (5 objects and 80 classes per 640 image)
+ for a, b, s in zip(m.cv4, m.cv5, m.stride): # from
+ a[-1].bias.data[:] = 1.0 # box
+ b[-1].bias.data[:m.nc] = math.log(5 / m.nc / (640 / s) ** 2) # cls (5 objects and 80 classes per 640 image)
+ for a, b, s in zip(m.cv6, m.cv7, m.stride): # from
+ a[-1].bias.data[:] = 1.0 # box
+ b[-1].bias.data[:m.nc] = math.log(5 / m.nc / (640 / s) ** 2) # cls (5 objects and 80 classes per 640 image)
+
+
+class TripleDDetect(nn.Module):
+ # YOLO Detect head for detection models
+ dynamic = False # force grid reconstruction
+ export = False # export mode
+ shape = None
+ anchors = torch.empty(0) # init
+ strides = torch.empty(0) # init
+
+ def __init__(self, nc=80, ch=(), inplace=True): # detection layer
+ super().__init__()
+ self.nc = nc # number of classes
+ self.nl = len(ch) // 3 # number of detection layers
+ self.reg_max = 16
+ self.no = nc + self.reg_max * 4 # number of outputs per anchor
+ self.inplace = inplace # use inplace ops (e.g. slice assignment)
+ self.stride = torch.zeros(self.nl) # strides computed during build
+
+ c2, c3 = make_divisible(max((ch[0] // 4, self.reg_max * 4, 16)), 4), \
+ max((ch[0], min((self.nc * 2, 128)))) # channels
+ c4, c5 = make_divisible(max((ch[self.nl] // 4, self.reg_max * 4, 16)), 4), \
+ max((ch[self.nl], min((self.nc * 2, 128)))) # channels
+ c6, c7 = make_divisible(max((ch[self.nl * 2] // 4, self.reg_max * 4, 16)), 4), \
+ max((ch[self.nl * 2], min((self.nc * 2, 128)))) # channels
+ self.cv2 = nn.ModuleList(
+ nn.Sequential(Conv(x, c2, 3), Conv(c2, c2, 3, g=4),
+ nn.Conv2d(c2, 4 * self.reg_max, 1, groups=4)) for x in ch[:self.nl])
+ self.cv3 = nn.ModuleList(
+ nn.Sequential(Conv(x, c3, 3), Conv(c3, c3, 3), nn.Conv2d(c3, self.nc, 1)) for x in ch[:self.nl])
+ self.cv4 = nn.ModuleList(
+ nn.Sequential(Conv(x, c4, 3), Conv(c4, c4, 3, g=4),
+ nn.Conv2d(c4, 4 * self.reg_max, 1, groups=4)) for x in ch[self.nl:self.nl*2])
+ self.cv5 = nn.ModuleList(
+ nn.Sequential(Conv(x, c5, 3), Conv(c5, c5, 3), nn.Conv2d(c5, self.nc, 1)) for x in ch[self.nl:self.nl*2])
+ self.cv6 = nn.ModuleList(
+ nn.Sequential(Conv(x, c6, 3), Conv(c6, c6, 3, g=4),
+ nn.Conv2d(c6, 4 * self.reg_max, 1, groups=4)) for x in ch[self.nl*2:self.nl*3])
+ self.cv7 = nn.ModuleList(
+ nn.Sequential(Conv(x, c7, 3), Conv(c7, c7, 3), nn.Conv2d(c7, self.nc, 1)) for x in ch[self.nl*2:self.nl*3])
+ self.dfl = DFL(self.reg_max)
+ self.dfl2 = DFL(self.reg_max)
+ self.dfl3 = DFL(self.reg_max)
+
+ def forward(self, x):
+ shape = x[0].shape # BCHW
+ d1 = []
+ d2 = []
+ d3 = []
+ for i in range(self.nl):
+ d1.append(torch.cat((self.cv2[i](x[i]), self.cv3[i](x[i])), 1))
+ d2.append(torch.cat((self.cv4[i](x[self.nl+i]), self.cv5[i](x[self.nl+i])), 1))
+ d3.append(torch.cat((self.cv6[i](x[self.nl*2+i]), self.cv7[i](x[self.nl*2+i])), 1))
+ if self.training:
+ return [d1, d2, d3]
+ elif self.dynamic or self.shape != shape:
+ self.anchors, self.strides = (d1.transpose(0, 1) for d1 in make_anchors(d1, self.stride, 0.5))
+ self.shape = shape
+
+ box, cls = torch.cat([di.view(shape[0], self.no, -1) for di in d1], 2).split((self.reg_max * 4, self.nc), 1)
+ dbox = dist2bbox(self.dfl(box), self.anchors.unsqueeze(0), xywh=True, dim=1) * self.strides
+ box2, cls2 = torch.cat([di.view(shape[0], self.no, -1) for di in d2], 2).split((self.reg_max * 4, self.nc), 1)
+ dbox2 = dist2bbox(self.dfl2(box2), self.anchors.unsqueeze(0), xywh=True, dim=1) * self.strides
+ box3, cls3 = torch.cat([di.view(shape[0], self.no, -1) for di in d3], 2).split((self.reg_max * 4, self.nc), 1)
+ dbox3 = dist2bbox(self.dfl3(box3), self.anchors.unsqueeze(0), xywh=True, dim=1) * self.strides
+ #y = [torch.cat((dbox, cls.sigmoid()), 1), torch.cat((dbox2, cls2.sigmoid()), 1), torch.cat((dbox3, cls3.sigmoid()), 1)]
+ #return y if self.export else (y, [d1, d2, d3])
+ y = torch.cat((dbox3, cls3.sigmoid()), 1)
+ return y if self.export else (y, d3)
+
+ def bias_init(self):
+ # Initialize Detect() biases, WARNING: requires stride availability
+ m = self # self.model[-1] # Detect() module
+ # cf = torch.bincount(torch.tensor(np.concatenate(dataset.labels, 0)[:, 0]).long(), minlength=nc) + 1
+ # ncf = math.log(0.6 / (m.nc - 0.999999)) if cf is None else torch.log(cf / cf.sum()) # nominal class frequency
+ for a, b, s in zip(m.cv2, m.cv3, m.stride): # from
+ a[-1].bias.data[:] = 1.0 # box
+ b[-1].bias.data[:m.nc] = math.log(5 / m.nc / (640 / s) ** 2) # cls (5 objects and 80 classes per 640 image)
+ for a, b, s in zip(m.cv4, m.cv5, m.stride): # from
+ a[-1].bias.data[:] = 1.0 # box
+ b[-1].bias.data[:m.nc] = math.log(5 / m.nc / (640 / s) ** 2) # cls (5 objects and 80 classes per 640 image)
+ for a, b, s in zip(m.cv6, m.cv7, m.stride): # from
+ a[-1].bias.data[:] = 1.0 # box
+ b[-1].bias.data[:m.nc] = math.log(5 / m.nc / (640 / s) ** 2) # cls (5 objects and 80 classes per 640 image)
+
+
+class Segment(Detect):
+ # YOLO Segment head for segmentation models
+ def __init__(self, nc=80, nm=32, npr=256, ch=(), inplace=True):
+ super().__init__(nc, ch, inplace)
+ self.nm = nm # number of masks
+ self.npr = npr # number of protos
+ self.proto = Proto(ch[0], self.npr, self.nm) # protos
+ self.detect = Detect.forward
+
+ c4 = max(ch[0] // 4, self.nm)
+ self.cv4 = nn.ModuleList(nn.Sequential(Conv(x, c4, 3), Conv(c4, c4, 3), nn.Conv2d(c4, self.nm, 1)) for x in ch)
+
+ def forward(self, x):
+ p = self.proto(x[0])
+ bs = p.shape[0]
+
+ mc = torch.cat([self.cv4[i](x[i]).view(bs, self.nm, -1) for i in range(self.nl)], 2) # mask coefficients
+ x = self.detect(self, x)
+ if self.training:
+ return x, mc, p
+ return (torch.cat([x, mc], 1), p) if self.export else (torch.cat([x[0], mc], 1), (x[1], mc, p))
+
+
+class DSegment(DDetect):
+ # YOLO Segment head for segmentation models
+ def __init__(self, nc=80, nm=32, npr=256, ch=(), inplace=True):
+ super().__init__(nc, ch[:-1], inplace)
+ self.nl = len(ch)-1
+ self.nm = nm # number of masks
+ self.npr = npr # number of protos
+ self.proto = Conv(ch[-1], self.nm, 1) # protos
+ self.detect = DDetect.forward
+
+ c4 = max(ch[0] // 4, self.nm)
+ self.cv4 = nn.ModuleList(nn.Sequential(Conv(x, c4, 3), Conv(c4, c4, 3), nn.Conv2d(c4, self.nm, 1)) for x in ch[:-1])
+
+ def forward(self, x):
+ p = self.proto(x[-1])
+ bs = p.shape[0]
+
+ mc = torch.cat([self.cv4[i](x[i]).view(bs, self.nm, -1) for i in range(self.nl)], 2) # mask coefficients
+ x = self.detect(self, x[:-1])
+ if self.training:
+ return x, mc, p
+ return (torch.cat([x, mc], 1), p) if self.export else (torch.cat([x[0], mc], 1), (x[1], mc, p))
+
+
+class DualDSegment(DualDDetect):
+ # YOLO Segment head for segmentation models
+ def __init__(self, nc=80, nm=32, npr=256, ch=(), inplace=True):
+ super().__init__(nc, ch[:-2], inplace)
+ self.nl = (len(ch)-2) // 2
+ self.nm = nm # number of masks
+ self.npr = npr # number of protos
+ self.proto = Conv(ch[-2], self.nm, 1) # protos
+ self.proto2 = Conv(ch[-1], self.nm, 1) # protos
+ self.detect = DualDDetect.forward
+
+ c6 = max(ch[0] // 4, self.nm)
+ c7 = max(ch[self.nl] // 4, self.nm)
+ self.cv6 = nn.ModuleList(nn.Sequential(Conv(x, c6, 3), Conv(c6, c6, 3), nn.Conv2d(c6, self.nm, 1)) for x in ch[:self.nl])
+ self.cv7 = nn.ModuleList(nn.Sequential(Conv(x, c7, 3), Conv(c7, c7, 3), nn.Conv2d(c7, self.nm, 1)) for x in ch[self.nl:self.nl*2])
+
+ def forward(self, x):
+ p = [self.proto(x[-2]), self.proto2(x[-1])]
+ bs = p[0].shape[0]
+
+ mc = [torch.cat([self.cv6[i](x[i]).view(bs, self.nm, -1) for i in range(self.nl)], 2),
+ torch.cat([self.cv7[i](x[self.nl+i]).view(bs, self.nm, -1) for i in range(self.nl)], 2)] # mask coefficients
+ d = self.detect(self, x[:-2])
+ if self.training:
+ return d, mc, p
+ return (torch.cat([d[0][1], mc[1]], 1), (d[1][1], mc[1], p[1]))
+
+
+class Panoptic(Detect):
+ # YOLO Panoptic head for panoptic segmentation models
+ def __init__(self, nc=80, sem_nc=93, nm=32, npr=256, ch=(), inplace=True):
+ super().__init__(nc, ch, inplace)
+ self.sem_nc = sem_nc
+ self.nm = nm # number of masks
+ self.npr = npr # number of protos
+ self.proto = Proto(ch[0], self.npr, self.nm) # protos
+ self.uconv = UConv(ch[0], ch[0]//4, self.sem_nc+self.nc)
+ self.detect = Detect.forward
+
+ c4 = max(ch[0] // 4, self.nm)
+ self.cv4 = nn.ModuleList(nn.Sequential(Conv(x, c4, 3), Conv(c4, c4, 3), nn.Conv2d(c4, self.nm, 1)) for x in ch)
+
+
+ def forward(self, x):
+ p = self.proto(x[0])
+ s = self.uconv(x[0])
+ bs = p.shape[0]
+
+ mc = torch.cat([self.cv4[i](x[i]).view(bs, self.nm, -1) for i in range(self.nl)], 2) # mask coefficients
+ x = self.detect(self, x)
+ if self.training:
+ return x, mc, p, s
+ return (torch.cat([x, mc], 1), p, s) if self.export else (torch.cat([x[0], mc], 1), (x[1], mc, p, s))
+
+
+class BaseModel(nn.Module):
+ # YOLO base model
+ def forward(self, x, profile=False, visualize=False):
+ return self._forward_once(x, profile, visualize) # single-scale inference, train
+
+ def _forward_once(self, x, profile=False, visualize=False):
+ y, dt = [], [] # outputs
+ for m in self.model:
+ if m.f != -1: # if not from previous layer
+ x = y[m.f] if isinstance(m.f, int) else [x if j == -1 else y[j] for j in m.f] # from earlier layers
+ if profile:
+ self._profile_one_layer(m, x, dt)
+ x = m(x) # run
+ y.append(x if m.i in self.save else None) # save output
+ if visualize:
+ feature_visualization(x, m.type, m.i, save_dir=visualize)
+ return x
+
+ def _profile_one_layer(self, m, x, dt):
+ c = m == self.model[-1] # is final layer, copy input as inplace fix
+ o = thop.profile(m, inputs=(x.copy() if c else x,), verbose=False)[0] / 1E9 * 2 if thop else 0 # FLOPs
+ t = time_sync()
+ for _ in range(10):
+ m(x.copy() if c else x)
+ dt.append((time_sync() - t) * 100)
+ if m == self.model[0]:
+ LOGGER.info(f"{'time (ms)':>10s} {'GFLOPs':>10s} {'params':>10s} module")
+ LOGGER.info(f'{dt[-1]:10.2f} {o:10.2f} {m.np:10.0f} {m.type}')
+ if c:
+ LOGGER.info(f"{sum(dt):10.2f} {'-':>10s} {'-':>10s} Total")
+
+ def fuse(self): # fuse model Conv2d() + BatchNorm2d() layers
+ LOGGER.info('Fusing layers... ')
+ for m in self.model.modules():
+ if isinstance(m, (RepConvN)) and hasattr(m, 'fuse_convs'):
+ m.fuse_convs()
+ m.forward = m.forward_fuse # update forward
+ if isinstance(m, (Conv, DWConv)) and hasattr(m, 'bn'):
+ m.conv = fuse_conv_and_bn(m.conv, m.bn) # update conv
+ delattr(m, 'bn') # remove batchnorm
+ m.forward = m.forward_fuse # update forward
+ self.info()
+ return self
+
+ def info(self, verbose=False, img_size=640): # print model information
+ model_info(self, verbose, img_size)
+
+ def _apply(self, fn):
+ # Apply to(), cpu(), cuda(), half() to model tensors that are not parameters or registered buffers
+ self = super()._apply(fn)
+ m = self.model[-1] # Detect()
+ if isinstance(m, (Detect, DualDetect, TripleDetect, DDetect, DualDDetect, TripleDDetect, Segment, DSegment, DualDSegment, Panoptic)):
+ m.stride = fn(m.stride)
+ m.anchors = fn(m.anchors)
+ m.strides = fn(m.strides)
+ # m.grid = list(map(fn, m.grid))
+ return self
+
+
+class DetectionModel(BaseModel):
+ # YOLO detection model
+ def __init__(self, cfg='yolo.yaml', ch=3, nc=None, anchors=None): # model, input channels, number of classes
+ super().__init__()
+ if isinstance(cfg, dict):
+ self.yaml = cfg # model dict
+ else: # is *.yaml
+ import yaml # for torch hub
+ self.yaml_file = Path(cfg).name
+ with open(cfg, encoding='ascii', errors='ignore') as f:
+ self.yaml = yaml.safe_load(f) # model dict
+
+ # Define model
+ ch = self.yaml['ch'] = self.yaml.get('ch', ch) # input channels
+ if nc and nc != self.yaml['nc']:
+ LOGGER.info(f"Overriding model.yaml nc={self.yaml['nc']} with nc={nc}")
+ self.yaml['nc'] = nc # override yaml value
+ if anchors:
+ LOGGER.info(f'Overriding model.yaml anchors with anchors={anchors}')
+ self.yaml['anchors'] = round(anchors) # override yaml value
+ self.model, self.save = parse_model(deepcopy(self.yaml), ch=[ch]) # model, savelist
+ self.names = [str(i) for i in range(self.yaml['nc'])] # default names
+ self.inplace = self.yaml.get('inplace', True)
+
+ # Build strides, anchors
+ m = self.model[-1] # Detect()
+ if isinstance(m, (Detect, DDetect, Segment, DSegment, Panoptic)):
+ s = 256 # 2x min stride
+ m.inplace = self.inplace
+ forward = lambda x: self.forward(x)[0] if isinstance(m, (Segment, DSegment, Panoptic)) else self.forward(x)
+ m.stride = torch.tensor([s / x.shape[-2] for x in forward(torch.zeros(1, ch, s, s))]) # forward
+ # check_anchor_order(m)
+ # m.anchors /= m.stride.view(-1, 1, 1)
+ self.stride = m.stride
+ m.bias_init() # only run once
+ if isinstance(m, (DualDetect, TripleDetect, DualDDetect, TripleDDetect, DualDSegment)):
+ s = 256 # 2x min stride
+ m.inplace = self.inplace
+ forward = lambda x: self.forward(x)[0][0] if isinstance(m, (DualDSegment)) else self.forward(x)[0]
+ m.stride = torch.tensor([s / x.shape[-2] for x in forward(torch.zeros(1, ch, s, s))]) # forward
+ # check_anchor_order(m)
+ # m.anchors /= m.stride.view(-1, 1, 1)
+ self.stride = m.stride
+ m.bias_init() # only run once
+
+ # Init weights, biases
+ initialize_weights(self)
+ self.info()
+ LOGGER.info('')
+
+ def forward(self, x, augment=False, profile=False, visualize=False):
+ if augment:
+ return self._forward_augment(x) # augmented inference, None
+ return self._forward_once(x, profile, visualize) # single-scale inference, train
+
+ def _forward_augment(self, x):
+ img_size = x.shape[-2:] # height, width
+ s = [1, 0.83, 0.67] # scales
+ f = [None, 3, None] # flips (2-ud, 3-lr)
+ y = [] # outputs
+ for si, fi in zip(s, f):
+ xi = scale_img(x.flip(fi) if fi else x, si, gs=int(self.stride.max()))
+ yi = self._forward_once(xi)[0] # forward
+ # cv2.imwrite(f'img_{si}.jpg', 255 * xi[0].cpu().numpy().transpose((1, 2, 0))[:, :, ::-1]) # save
+ yi = self._descale_pred(yi, fi, si, img_size)
+ y.append(yi)
+ y = self._clip_augmented(y) # clip augmented tails
+ return torch.cat(y, 1), None # augmented inference, train
+
+ def _descale_pred(self, p, flips, scale, img_size):
+ # de-scale predictions following augmented inference (inverse operation)
+ if self.inplace:
+ p[..., :4] /= scale # de-scale
+ if flips == 2:
+ p[..., 1] = img_size[0] - p[..., 1] # de-flip ud
+ elif flips == 3:
+ p[..., 0] = img_size[1] - p[..., 0] # de-flip lr
+ else:
+ x, y, wh = p[..., 0:1] / scale, p[..., 1:2] / scale, p[..., 2:4] / scale # de-scale
+ if flips == 2:
+ y = img_size[0] - y # de-flip ud
+ elif flips == 3:
+ x = img_size[1] - x # de-flip lr
+ p = torch.cat((x, y, wh, p[..., 4:]), -1)
+ return p
+
+ def _clip_augmented(self, y):
+ # Clip YOLO augmented inference tails
+ nl = self.model[-1].nl # number of detection layers (P3-P5)
+ g = sum(4 ** x for x in range(nl)) # grid points
+ e = 1 # exclude layer count
+ i = (y[0].shape[1] // g) * sum(4 ** x for x in range(e)) # indices
+ y[0] = y[0][:, :-i] # large
+ i = (y[-1].shape[1] // g) * sum(4 ** (nl - 1 - x) for x in range(e)) # indices
+ y[-1] = y[-1][:, i:] # small
+ return y
+
+
+Model = DetectionModel # retain YOLO 'Model' class for backwards compatibility
+
+
+class SegmentationModel(DetectionModel):
+ # YOLO segmentation model
+ def __init__(self, cfg='yolo-seg.yaml', ch=3, nc=None, anchors=None):
+ super().__init__(cfg, ch, nc, anchors)
+
+
+class ClassificationModel(BaseModel):
+ # YOLO classification model
+ def __init__(self, cfg=None, model=None, nc=1000, cutoff=10): # yaml, model, number of classes, cutoff index
+ super().__init__()
+ self._from_detection_model(model, nc, cutoff) if model is not None else self._from_yaml(cfg)
+
+ def _from_detection_model(self, model, nc=1000, cutoff=10):
+ # Create a YOLO classification model from a YOLO detection model
+ if isinstance(model, DetectMultiBackend):
+ model = model.model # unwrap DetectMultiBackend
+ model.model = model.model[:cutoff] # backbone
+ m = model.model[-1] # last layer
+ ch = m.conv.in_channels if hasattr(m, 'conv') else m.cv1.conv.in_channels # ch into module
+ c = Classify(ch, nc) # Classify()
+ c.i, c.f, c.type = m.i, m.f, 'models.common.Classify' # index, from, type
+ model.model[-1] = c # replace
+ self.model = model.model
+ self.stride = model.stride
+ self.save = []
+ self.nc = nc
+
+ def _from_yaml(self, cfg):
+ # Create a YOLO classification model from a *.yaml file
+ self.model = None
+
+
+def parse_model(d, ch): # model_dict, input_channels(3)
+ # Parse a YOLO model.yaml dictionary
+ LOGGER.info(f"\n{'':>3}{'from':>18}{'n':>3}{'params':>10} {'module':<40}{'arguments':<30}")
+ anchors, nc, gd, gw, act = d['anchors'], d['nc'], d['depth_multiple'], d['width_multiple'], d.get('activation')
+ if act:
+ Conv.default_act = eval(act) # redefine default activation, i.e. Conv.default_act = nn.SiLU()
+ RepConvN.default_act = eval(act) # redefine default activation, i.e. Conv.default_act = nn.SiLU()
+ LOGGER.info(f"{colorstr('activation:')} {act}") # print
+ na = (len(anchors[0]) // 2) if isinstance(anchors, list) else anchors # number of anchors
+ no = na * (nc + 5) # number of outputs = anchors * (classes + 5)
+
+ layers, save, c2 = [], [], ch[-1] # layers, savelist, ch out
+ for i, (f, n, m, args) in enumerate(d['backbone'] + d['head']): # from, number, module, args
+ m = eval(m) if isinstance(m, str) else m # eval strings
+ for j, a in enumerate(args):
+ with contextlib.suppress(NameError):
+ args[j] = eval(a) if isinstance(a, str) else a # eval strings
+
+ n = n_ = max(round(n * gd), 1) if n > 1 else n # depth gain
+ if m in {
+ Conv, AConv, ConvTranspose,
+ Bottleneck, SPP, SPPF, DWConv, BottleneckCSP, nn.ConvTranspose2d, DWConvTranspose2d, SPPCSPC, ADown,
+ ELAN1, RepNCSPELAN4, SPPELAN}:
+ c1, c2 = ch[f], args[0]
+ if c2 != no: # if not output
+ c2 = make_divisible(c2 * gw, 8)
+
+ args = [c1, c2, *args[1:]]
+ if m in {BottleneckCSP, SPPCSPC}:
+ args.insert(2, n) # number of repeats
+ n = 1
+ elif m is nn.BatchNorm2d:
+ args = [ch[f]]
+ elif m is Concat:
+ c2 = sum(ch[x] for x in f)
+ elif m is Shortcut:
+ c2 = ch[f[0]]
+ elif m is ReOrg:
+ c2 = ch[f] * 4
+ elif m is CBLinear:
+ c2 = args[0]
+ c1 = ch[f]
+ args = [c1, c2, *args[1:]]
+ elif m is CBFuse:
+ c2 = ch[f[-1]]
+ # TODO: channel, gw, gd
+ elif m in {Detect, DualDetect, TripleDetect, DDetect, DualDDetect, TripleDDetect, Segment, DSegment, DualDSegment, Panoptic}:
+ args.append([ch[x] for x in f])
+ # if isinstance(args[1], int): # number of anchors
+ # args[1] = [list(range(args[1] * 2))] * len(f)
+ if m in {Segment, DSegment, DualDSegment, Panoptic}:
+ args[2] = make_divisible(args[2] * gw, 8)
+ elif m is Contract:
+ c2 = ch[f] * args[0] ** 2
+ elif m is Expand:
+ c2 = ch[f] // args[0] ** 2
+ else:
+ c2 = ch[f]
+
+ m_ = nn.Sequential(*(m(*args) for _ in range(n))) if n > 1 else m(*args) # module
+ t = str(m)[8:-2].replace('__main__.', '') # module type
+ np = sum(x.numel() for x in m_.parameters()) # number params
+ m_.i, m_.f, m_.type, m_.np = i, f, t, np # attach index, 'from' index, type, number params
+ LOGGER.info(f'{i:>3}{str(f):>18}{n_:>3}{np:10.0f} {t:<40}{str(args):<30}') # print
+ save.extend(x % i for x in ([f] if isinstance(f, int) else f) if x != -1) # append to savelist
+ layers.append(m_)
+ if i == 0:
+ ch = []
+ ch.append(c2)
+ return nn.Sequential(*layers), sorted(save)
+
+
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--cfg', type=str, default='yolo.yaml', help='model.yaml')
+ parser.add_argument('--batch-size', type=int, default=1, help='total batch size for all GPUs')
+ parser.add_argument('--device', default='', help='cuda device, i.e. 0 or 0,1,2,3 or cpu')
+ parser.add_argument('--profile', action='store_true', help='profile model speed')
+ parser.add_argument('--line-profile', action='store_true', help='profile model speed layer by layer')
+ parser.add_argument('--test', action='store_true', help='test all yolo*.yaml')
+ opt = parser.parse_args()
+ opt.cfg = check_yaml(opt.cfg) # check YAML
+ print_args(vars(opt))
+ device = select_device(opt.device)
+
+ # Create model
+ im = torch.rand(opt.batch_size, 3, 640, 640).to(device)
+ model = Model(opt.cfg).to(device)
+ model.eval()
+
+ # Options
+ if opt.line_profile: # profile layer by layer
+ model(im, profile=True)
+
+ elif opt.profile: # profile forward-backward
+ results = profile(input=im, ops=[model], n=3)
+
+ elif opt.test: # test all models
+ for cfg in Path(ROOT / 'models').rglob('yolo*.yaml'):
+ try:
+ _ = Model(cfg)
+ except Exception as e:
+ print(f'Error in {cfg}: {e}')
+
+ else: # report fused model summary
+ model.fuse()